sqoop导数脚本

从hive导数到mysql

#!/bin/bash
#date:20190827
OIFS=$IFS
#当前路径
baseDirForScriptSelf=$(cd "$(dirname "$0")"; pwd)
#本地路径
localPath=${baseDirForScriptSelf}
#java文件路径
java_path=${localPath}/java
mkdir -p ${java_path}
#日志文件路径
cur_date=`date -d "0 days ago" +%Y%m%d`
log_dir=${baseDirForScriptSelf}/log/${cur_date}
if [ ! -d ${log_dir} ]
then 
mkdir -p ${log_dir}
else 
echo "文件夹已存在"
fi

############################下面还有#################################
#并行的计数器
syn_num=0
#并行的个数
syn_num_pv=5
#获取控制文件中参数
maxLog=1
function log_increase(){
log_dir=$1
log_name=$2
maxLog=`ls -r ${log_dir}|grep -v "^d"|grep ${log_name}|wc -l`
let maxLog=maxLog+1
}


function sqoop_mon(){
        echo "执行脚本并行度.......${syn_num_pv}"
		echo "hive测试数据库.......${database}"
		echo "mysql测试地址.........${jdbcUrl}"
		echo "mysql测试用户名.....${username}"
		echo "mysql测试密码.${password}"
		echo "hive仓库测试地址...${exportDir}"
		echo "mysql目标表...${table_nm_mysql}"
		echo "hive源表...${table_nm_hive}"
#sqoop export --connect ${jdbcUrl} --username ${username} --password ${password} --table ${table_nm_mysql} --input-null-string '\\N' --input-null-non-string '\\N' --fields-terminated-by '\001' --export-dir ${exportDir}/${table_nm_hive} --m 1
#sqoop export --connect jdbc:mysql://10.1.100.2:3306/zkr --username zkr --password Zkr@1234 --table back_lcappnt --input-null-string '\\N' --input-null-non-string '\\N' --fields-terminated-by '\001' --export-dir /user/hive/warehouse/dc_sma.db/lis_lcappnt_tmp_zkr --m 1
}
function gettimeFlag(){
h=`date +%H`

hh=""
if [ $h -ge 0 ] && [ $h -lt 12 ];
   then
       hh=am
   else
       hh=pm
fi

}

function main(){
mon_count=$1
IFS=$'\n'
for line in `cat ${localPath}/conf/hive.properties`
   do
   IFS=$'\n'
   table_flag=${line%%:*}
   echo $table_flag
   if [ x$table_flag == x"syn_num_pv" ];then
       syn_num_pv=${line%%\#*}
	   syn_num_pv=${syn_num_pv:11}
   fi
   if [ x$table_flag == x"database" ];then
       database=${line%%\#*}
	   database=${database:9}
   fi
   if [ x$table_flag == x"mysqlurl" ];then
       mysqlurl=${line%%\#*}
	   mysqlurl=${mysqlurl:9}
   fi
   if [ x$table_flag == x"username" ];then
       username=${line%%\#*}
	   username=${username:9}
   fi
   if [ x$table_flag == x"password" ];then
       password=${line%%\#*}
	   password=${password:9}
   fi
   if [ x$table_flag == x"exportDir" ];then
       exportDir=${line%%\#*}
	   exportDir=${exportDir:10}
   fi
   if [ x$table_flag == x"exe_table_hive" ];
      then
	      table_nm_hive=${line%%\#*}
	      table_nm_hive=${table_nm_hive:15}
		  line=${line#*\#}
	      mysql_table=${line%%:*}
		  table_nm_mysql=""
		  if [ x${mysql_table} == x"exe_table_mysql" ];
			then
                table_nm_mysql=${line%%\#*}
	            table_nm_mysql=${table_nm_mysql:16}
            fi
      syn_num=`expr $syn_num + 1`
      echo `date +%Y-%m-%d,%H:%M`:"执行主函数"
            jdbcUrl=${mysqlurl}	
			upper=${table_nm}		  
        #显示当前时间
        cur_date=`date -d "0 days ago" +%Y%m%d`
	   #echo "--------------------------------$sqlcondition"
	   #echo "..................................${hadoopIP} ${database} ${table_nm} ${jdbcUrl} ${username} ${passwd}"
	  log_name=${table_nm}_${cur_date}
	  log_increase "${log_dir}" "${log_name}"
	  fial_log_name=${log_name}_${maxLog}.log
	  gettimeFlag
	  time=${cur_date}$hh
	  #sqoop export 核心方法
	  sqoop_mon 2>&1 |tee -a  ${log_dir}/${fial_log_name} &
	  if (( $syn_num % $syn_num_pv == 0 ))
			then
			wait
			echo "ok......"
	  fi
	fi
done
}

wait 


h_count=1
main "${h_count}"
echo `date +%Y-%m-%d,%H:%M`:"当前主函数执行完毕！！！"

#执行脚本并行度
syn_num_pv:3#
#hive测试数据库
database:dc_sma#
#mysql测试地址
mysqlurl:jdbc:mysql://10.1.100.2:3306/zkr#
#mysql测试用户名
username:#
#mysql测试密码
password:#
#hive仓库测试地址
exportDir:/user/hive/warehouse/dc_sma.db#
#需要export的表
exe_table_hive:lis_lcaddress_tmp_zkr#exe_table_mysql:back_lcaddress#
exe_table_hive:lis_lcappnt_tmp_zkr#exe_table_mysql:back_lcappnt#
exe_table_hive:lis_lccont_tmp_zkr#exe_table_mysql:back_lccont#