hadoop distcp 脚本

#!/bin/bash
source_path='hdfs://harunasg/home/xxx/date=${date}'
target_path='hdfs://harunava/home/xxx/date=${date}'


source="${source_path}"
target="${target_path}"
echo "start handle path: $path"
hadoop fs -rm -r "$target"
hadoop fs -mkdir -p "$target"
if [ $? -ne 0 ]; then
    echo "mkdir path error, path: $target"
    exit 1
fi
echo "finish mkdir path: $target"
echo "start distcp data, path: $target"
hadoop distcp \
-Dyarn.cluster.name=mouse \
-Dmapred.job.queue.name=root.mouse_dp_security_dr \
-Dmapreduce.job.name=sync_dm_discovery_table_daily_sg_to_va_tangliang \
-Dmapreduce.map.memory.mb=4000 \
-Dmapreduce.map.cpu.vcores=1 \
-Dmapreduce.reduce.memory.mb=4000 \
-Dmapreduce.reduce.cpu.vcores=1 \
-bandwidth=25 \
-m 800 \
-skipcrccheck \
-overwrite "$source" "$target"
res=$?
if [ "${res}" -ne 0 ]; then
    echo "hadoop distcp failed. status: ${res}, path: $target"
    exit 1
fi
echo "hadoop distcp success, path: $target"
hadoop fs -du -s "$source"
hadoop fs -du -s "$target"
hadoop fs -touchz ${target_path}/_MOVE_SUCCESS