叢集間傳輸資料
阿新 • • 發佈:2018-12-02
#!/bin/bash
# data train 1.0
# CREATED BY ZJF @2018-08-01
# source env
source /etc/profile
export JAVA_HOME=/home/work/soft/jdk1.7.0_55
# args check
if [ $# -eq 2 ]
then
echo "the args is right, caculating..."
else
echo -e "\n\nERROR! \nthe args is wrong ,you should give 2 params, one aim hdfs path and anothor destination hdfs path, \nlike '/user/h_data_platform/platform/gamesdk/dwm_gc_v2_comic_action/date=2018-08-01/ /user/h_data_platform/platform/gamesdk/dw
m_gc_v2_comic_action/date=2018-08-01/'\n\n"
exit 1;
fi
# set workdir
WORK_DIR=$(cd `dirname $0`; pwd)
echo ${WORK_DIR}
cd ${WORK_DIR}
# transfor data between clusters
# use infra-client, must.
/home/work/soft/infra-client/bin/hadoop --cluster zjyprc-hadoop fs -mkdir -p $2; /home/work/soft/infra-client/bin/hadoop --cluster zjyprc-hadoop distcp -D mapreduce.job.queuename=root.production.game_group.gamepublish.game_publish_queue -prpc -m 50 -ba
ndwidth 1 -ignoreDeleted -overwrite hdfs://c3prc-hadoop${1} hdfs://zjyprc-hadoop${2}