sqoop 从mysql oracle 数据库导入数据 到 hive

定时 增量 导入数据

sqoop job --create sqoop_job_incremental -- import --connect jdbc:mysql://192.168.1.87:3306/world --username root --password 123456 --table sqoop_job_incremental --hive-import --hive-table sqoop_job_incremental --check-column last_mod --incremental append --fields-terminated-by ',' -m 1
 

编写 shell 脚本，添加权限

#!/bin/sh
source /etc/profile
sqoop job --exec  sqoop_job_incremental3
 

chmod -r 777 /home/hadoop/test.sh

 

crontab –e 编写一条定时任务 */5 * * * * /home/test.sh 在每5分钟执行一次test.sh脚本

 */5 * * * * /home/test.sh 

 

 

*/5 * * * *  ./etc/profile; /home/hadoop/test.sh
 

 

 

---通过Oracle 创建hive 表结构 MONITORDATE
sqoop create-hive-table --connect jdbc:oracle:thin:@192.168.1.87:1521:ORCL --username ONEPLATFORM --password ONEPLATFORM --table AAIRAUTOITEM

---
sqoop import --connect jdbc:mysql://192.168.1.87:3306/world --username root --password 123456 --table sqoop_job --hive-import --hive-table sqoop_job --check-column id --incremental append --last-value 3 --fields-terminated-by ','  -m 1

--
sqoop import --connect jdbc:mysql://192.168.1.87:3306/world --username root --password 123456 --query 'select * from city where $CONDITIONS LIMIT 100' --target-dir /user/root/sqoop/city --delete-target-dir --num-mappers 1 --compress --compression-codec org.apache.hadoop.io.compress.SnappyCodec --direct --fields-terminated-by '\t'
---
sqoop import --connect jdbc:oracle:thin:@192.168.1.87:1521:ORCL --username ONEPLATFORM --password ONEPLATFORM  --table YQDATADAY   

sqoop import --connect jdbc:mysql://192.168.1.87:3306/world --username root --password 123456 --table city -m 1

sqoop list-tables --connect jdbc:mysql://192.168.1.87:3306/world --username root --password 123456
---
sqoop list-databases --connect jdbc:mysql://192.168.1.87:3306 --username root --password 123456
---
sqoop list-databases --connect jdbc:oracle:thin:192.168.1.87:ONEPLATFORM  --username ONEPLATFORM --password ONEPLATFORM