sqoop 任务配置

前提 ：sqoop 的元数据存储在mysql 里面 0.0.0.1

1：tws_order

MySQL 中的tws增量数据到hive缓存表

sqoop-job \
--create tws_order_add_job \
--meta-connect jdbc:hsqldb:hsql://0.0.0.1:16000/sqoop \
-- import \
--connect "jdbc:mysql://0.0.0.19:3306/guangqidb?useUnicode=true&characterEncoding=utf-8&useSSL=false" \
--username root \
--password-file /user/sqoop/sqoop.pwd \
--query "select * from tws_order where 1=1 and \$CONDITIONS" \
--split-by uid --fields-terminated-by '\001' \
--lines-terminated-by '\n' \
--target-dir hdfs://0.0.0.18:8020/user/hive/warehouse/temporary_tables.db/ods_order_temporary  \
--incremental append \
--check-column uid \
--last-value 0

dm数据到19 mysql数据库：

通过字段data_date,org_code,factory_code,factory_city_code,car_brand_code 对数据做updata操作

sqoop-job \
-Doraoop.nologging=true \
-Dsqoop.export.records.per.statement=1 \
-Dsqoop.export.statements.per.transaction=1 \
--create dm_tws_order_mysql_job \
--meta-connect jdbc:hsqldb:hsql://0.0.0.1:16000/sqoop \
-- export \
--connect "jdbc:mysql://0.0.0.19:3306/guangqidb?useUnicode=true&characterEncoding=utf-8&useSSL=false" \
--username root \
--password-file /user/sqoop/sqoop.pwd \
--table dm_order_day \
--export-dir /user/hive/warehouse/dm.db/dm_order_day \
--update-key data_date,org_code,factory_code,factory_city_code,car_brand_code \
--update-mode allowinsert \
--input-fields-terminated-by '\001'

dwd数据到19 mysql数据库

通过字段org_code,order_id,customer_order_id,id 对数据做updata操作

sqoop-job \
-Doraoop.nologging=true \
-Dsqoop.export.records.per.statement=1 \
-Dsqoop.export.statements.per.transaction=1 \
--create dwd_tws_order_mysql_job \
--meta-connect jdbc:hsqldb:hsql://0.0.0.1:16000/sqoop \
-- export \
--connect "jdbc:mysql://0.0.0.19:3306/guangqidb?useUnicode=true&characterEncoding=utf-8&useSSL=false" \
--username root \
--password-file /user/sqoop/sqoop.pwd \
--table dwd_order \
--export-dir /user/hive/warehouse/dwd.db/dwd_order \
--update-key org_code,order_id,customer_order_id,id \
--update-mode allowinsert \
--input-fields-terminated-by '\001'

执行命令

job --meta-connect jdbc:hsqldb:hsql://0.0.0.1:16000/sqoop --exec tws_order_add_job

job --meta-connect jdbc:hsqldb:hsql://0.0.0.1:16000/sqoop --exec dm_tws_order_mysql_job

job --meta-connect jdbc:hsqldb:hsql://0.0.0.1:16000/sqoop --exec dwd_tws_order_mysql_job