从hive导数到mysql
#!/bin/bash
#date:20190827
OIFS=$IFS
#当前路径
baseDirForScriptSelf=$(cd "$(dirname "$0")"; pwd)
#本地路径
localPath=${baseDirForScriptSelf}
#java文件路径
java_path=${localPath}/java
mkdir -p ${java_path}
#日志文件路径
cur_date=`date -d "0 days ago" +%Y%m%d`
log_dir=${baseDirForScriptSelf}/log/${cur_date}
if [ ! -d ${log_dir} ]
then
mkdir -p ${log_dir}
else
echo "文件夹已存在"
fi
############################下面还有#################################
#并行的计数器
syn_num=0
#并行的个数
syn_num_pv=5
#获取控制文件中参数
maxLog=1
function log_increase(){
log_dir=$1
log_name=$2
maxLog=`ls -r ${log_dir}|grep -v "^d"|grep ${log_name}|wc -l`
let maxLog=maxLog+1
}
function sqoop_mon(){
echo "执行脚本并行度.......${syn_num_pv}"
echo "hive测试数据库.......${database}"
echo "mysql测试地址.........${jdbcUrl}"
echo "mysql测试用户名.....${username}"
echo "mysql测试密码.${password}"
echo "hive仓库测试地址...${exportDir}"
echo "mysql目标表...${table_nm_mysql}"
echo "hive源表...${table_nm_hive}"
#sqoop export --connect ${jdbcUrl} --username ${username} --password ${password} --table ${table_nm_mysql} --input-null-string '\\N' --input-null-non-string '\\N' --fields-terminated-by '\001' --export-dir ${exportDir}/${table_nm_hive} --m 1
#sqoop export --connect jdbc:mysql://10.1.100.2:3306/zkr --username zkr --password Zkr@1234 --table back_lcappnt --input-null-string '\\N' --input-null-non-string '\\N' --fields-terminated-by '\001' --export-dir /user/hive/warehouse/dc_sma.db/lis_lcappnt_tmp_zkr --m 1
}
function gettimeFlag(){
h=`date +%H`
hh=""
if [ $h -ge 0 ] && [ $h -lt 12 ];
then
hh=am
else
hh=pm
fi
}
function main(){
mon_count=$1
IFS=$'\n'
for line in `cat ${localPath}/conf/hive.properties`
do
IFS=$'\n'
table_flag=${line%%:*}
echo $table_flag
if [ x$table_flag == x"syn_num_pv" ];then
syn_num_pv=${line%%\#*}
syn_num_pv=${syn_num_pv:11}
fi
if [ x$table_flag == x"database" ];then
database=${line%%\#*}
database=${database:9}
fi
if [ x$table_flag == x"mysqlurl" ];then
mysqlurl=${line%%\#*}
mysqlurl=${mysqlurl:9}
fi
if [ x$table_flag == x"username" ];then
username=${line%%\#*}
username=${username:9}
fi
if [ x$table_flag == x"password" ];then
password=${line%%\#*}
password=${password:9}
fi
if [ x$table_flag == x"exportDir" ];then
exportDir=${line%%\#*}
exportDir=${exportDir:10}
fi
if [ x$table_flag == x"exe_table_hive" ];
then
table_nm_hive=${line%%\#*}
table_nm_hive=${table_nm_hive:15}
line=${line#*\#}
mysql_table=${line%%:*}
table_nm_mysql=""
if [ x${mysql_table} == x"exe_table_mysql" ];
then
table_nm_mysql=${line%%\#*}
table_nm_mysql=${table_nm_mysql:16}
fi
syn_num=`expr $syn_num + 1`
echo `date +%Y-%m-%d,%H:%M`:"执行主函数"
jdbcUrl=${mysqlurl}
upper=${table_nm}
#显示当前时间
cur_date=`date -d "0 days ago" +%Y%m%d`
#echo "--------------------------------$sqlcondition"
#echo "..................................${hadoopIP} ${database} ${table_nm} ${jdbcUrl} ${username} ${passwd}"
log_name=${table_nm}_${cur_date}
log_increase "${log_dir}" "${log_name}"
fial_log_name=${log_name}_${maxLog}.log
gettimeFlag
time=${cur_date}$hh
#sqoop export 核心方法
sqoop_mon 2>&1 |tee -a ${log_dir}/${fial_log_name} &
if (( $syn_num % $syn_num_pv == 0 ))
then
wait
echo "ok......"
fi
fi
done
}
wait
h_count=1
main "${h_count}"
echo `date +%Y-%m-%d,%H:%M`:"当前主函数执行完毕!!!"
#执行脚本并行度
syn_num_pv:3#
#hive测试数据库
database:dc_sma#
#mysql测试地址
mysqlurl:jdbc:mysql://10.1.100.2:3306/zkr#
#mysql测试用户名
username:#
#mysql测试密码
password:#
#hive仓库测试地址
exportDir:/user/hive/warehouse/dc_sma.db#
#需要export的表
exe_table_hive:lis_lcaddress_tmp_zkr#exe_table_mysql:back_lcaddress#
exe_table_hive:lis_lcappnt_tmp_zkr#exe_table_mysql:back_lcappnt#
exe_table_hive:lis_lccont_tmp_zkr#exe_table_mysql:back_lccont#