sqoop导数

#!/bin/bash

source ExitCodeCheck.sh
opts=$@
getparam(){
    echo $opts|xargs -n1|cut -b 2-|awk -v arg=$1 -F'=' '$1==arg{print $2}'
}

IncStart=`getparam inc_start`
IncEnd=`getparam inc_end`
db_connection=`getparam jdbc_str`
db_username=`getparam db_user`
db_password=`getparam db_psw`
dataName=`getparam db_sid`
queueName=`getparam hdp_queue`
hdfshostname=`getparam hdfs_host`
hduser=`getparam hdp_user_name`

IncStartYear=${IncStart:0:4}
IncStartMonth=${IncStart:4:2}
IncStartDay=${IncStart:6:2}
IncStartAll=${IncStartYear}"-"${IncStartMonth}"-"${IncStartDay}" 00:00:00.0"
IncEndYear=${IncEnd:0:4}
IncEndMonth=${IncEnd:4:2}
IncEndDay=${IncEnd:6:2}
IncEndAll=${IncEndYear}"-"${IncEndMonth}"-"${IncEndDay}" 00:00:00.0"
IncStartFormat=${IncStartYear}"-"${IncStartMonth}"-"${IncStartDay}

job_name=${0}_$$

hive_db_name=sx_bdp_smdb_safe
hive_db_user=hduser1519

#源表 
src_table=smdbwsdata.ashareblocktrade
#初始化临时表
idld_table=idld_smdbds_ashareblocktrade
#目标表
hive_table=bas_smdbds_ashareblocktrade
#导出临时目录
target_dir=/apps-data/${hive_db_user}/${hive_db_name}/import/${idld_table} 
#支持二次运行
if [[ $target_dir =~ ^/apps-data/${hive_db_user}/${hive_db_name}/import/[a-zA-Z0-9_]+ ]];then
    hadoop fs -rm -r $target_dir
fi

sqoop import -D mapred.job.queue.name=${queueName} -D mapred.job.name=${job_name} \
--connect ${db_connection} \
--username ${db_username}  \
--password ${db_password}  \
--query "
select 
    object_id
    ,s_info_windcode
    ,trade_dt
    ,s_block_price
    ,s_block_volume
    ,s_block_amount
    ,crncy_code
    ,s_block_buyername
    ,s_block_sellername
    ,s_block_frequency
    ,opdate
    ,opmode
    ,created_by
    ,created_date
    ,updated_by
    ,updated_date
    ,'hduser1519' creator
    ,sysdate cdate
    ,'hduser1519' updator
    ,sysdate udate
  from $src_table
 where 1=1 and \$CONDITIONS
        " \
-m 1                                                               \
--hive-table ${hive_db_name}.${idld_table}                         \
--hive-drop-import-delims                                          \
--fetch-size 5000                                                  \
--target-dir "${target_dir}"   \
--hive-overwrite         \
--null-string '\\N'      \
--null-non-string '\\N'  \
--hive-import;           

exitCodeCheck $?

hive -v -e "
use ${hive_db_name};
set mapred.job.queue.name=${queueName};
set mapred.job.name=${job_name};


insert overwrite TABLE ${hive_db_name}.${hive_table} 
select 
    object_id
    ,s_info_windcode
    ,trade_dt
    ,s_block_price
    ,s_block_volume
    ,s_block_amount
    ,crncy_code
    ,s_block_buyername
    ,s_block_sellername
    ,s_block_frequency
    ,opdate
    ,opmode
    ,created_by
    ,created_date
    ,updated_by
    ,updated_date
    ,creator
    ,cdate
    ,updator
    ,udate
  from ${hive_db_name}.$idld_table 
;

"
exitCodeCheck $?

转载于:https://www.cnblogs.com/koushr/p/9483777.html