parquet to mysql_hdfs TEXTFILE和PARQUET 格式推送数据到mysql

首先清理mysql目标表

sqoop eval -driver com.mysql.jdbc.Driver \

--connect jdbc:mysql://****:3306/basicdata?characterEncoding=UTF-8 \

--username root \

--password '****' \

--query "TRUNCATE TABLE DAILY_RATES"

textfile文件推送

sqoop export \

--driver com.mysql.jdbc.Driver \

--connect jdbc:mysql://****:3306/basicdata?characterEncoding=UTF-8 \

--table DAILY_RATES  \

--username root \

--password '****' \

--input-null-string '\\N' \

--input-null-non-string '\\N' \

--fields-terminated-by '\001' \

--export-dir /DHDC/DWD/DWD_DAILY_RATES \

--m 1

PARQUET 文件推送(这里impala表示分区的，所以我现获取了最大的分区时间)

just_time=$(impala-shell -u hive -q 'select max(dt) from DHDC_DWD.DWD_STD_INV_ORG' -B)

echo $just_time

sqoop export --driver com.mysql.jdbc.Driver --connect jdbc:mysql://***:3306/basicdata?characterEncoding=UTF-8 --table STD_INV_ORG  --username root --password '****' --columns "inv_org_id ,   inv_org_short_code,inv_org_cn,inv_org_en,ou_short_code_fk,date_from,date_to,create_user,create_time,update_user,update_time,dt" --input-null-string '\\N' --input-null-non-string '\\N' --fields-terminated-by '\001' --hcatalog-database DHDC_DWD --hcatalog-table DWD_STD_INV_ORG --hcatalog-partition-keys dt   --hcatalog-partition-values $just_time --num-mappers 8

#!/bin/sh

source /etc/profile

source ~/.bash_profile

just_time=$(impala-shell -u hive -q 'select max(dt) from DHDC_DWD.DWD_DAILY_RATES' -B)

echo $just_time

sqoop export --driver com.mysql.jdbc.Driver --connect jdbc:mysql://***:3306/basicdata?characterEncoding=UTF-8 --table DAILY_RATES --username root --password '***' --columns "DAILY_RATES_ID,FROM_CURRENCY,TO_CURRENCY,CONVERSION_DATE,CONVERSION_TYPE,CONVERSION_RATE,INVERSE_CON_RATE ,CREATE_USER,CREATE_TIME,UPDATE_USER,UPDATE_TIME,DT" --input-null-string '\\N' --input-null-non-string '\\N' --fields-terminated-by '\001' --hcatalog-database DHDC_DWD --hcatalog-table DWD_DAILY_RATES --hcatalog-partition-keys dt --hcatalog-partition-values $just_time --num-mappers 8

[[ $? -eq 0 ]]||exit

#数据推送成功后，向版本里，写入当前版本

sqoop eval --connect jdbc:mysql://***:3306/basicdata?characterEncoding=UTF-8 --username root --password '***' --e "INSERT into VERSION(table_name,dt) values('DWD_DAILY_RATES','$just_time')"