pgSQL数据导入到Hbase，hive

pgSQL-->hbase

1-hbase shell

2-hbase建表:
create 'hb_init_shop','detail'

3-sftp
复制Python脚本
cd /home/bigdata/bigdata_scripts/bigdata/scripts/bin

sftp> put -r "/Users/MP/Desktop/bigdata/scripts/binimport_hb_init_market_card_info.py"

4-执行Python脚本
 bin]$ python import_hb_init_market_card_info.py -1

5- 查看hbase的表
hbase(main):002:0> list

6-查看导入的数据
hbase(main):004:0> scan 'hb_init_market_card_user_info'

cd $SQOOP_HOME ;  

 bin/sqoop import  --connect jdbc:postgresql://172.16.15.34:5432/t  --username postgres  --password cX98Ugt  --query "SELECT id, park_id, car_number, in_time, out_time, duration, park_amount, finish, mid, sid, gid, order_no, ali_user_id, ali_parking_id, ctime, mtime FROM admin.ali_park_info where mtime >= '1970-01-01' and  mtime < '2018-07-16'  and \$CONDITIONS"  --split-by id  --hbase-table hb_init_app_god_detail  --column-family detail  --hbase-row-key id  --null-string  '\\N'  --null-non-string '\\N' 

-->hive

drop table default.ods_hive_shop; 
create external table default.ods_hive_app_god_detail(key string,id string,park_id string, car_number string, in_time string, out_time string, duration string, park_amount string, finish string, mid string, sid string, gid string, order_no string, ali_user_id string, ali_parking_id string, ctime string, mtime  string) 
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' 
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,detail:id,detail:park_id, detail:car_number, detail:in_time, detail:out_time, detail:duration, detail:park_amount, detail:finish, detail:mid, detail:sid, detail:gid, detail:order_no, detail:ali_user_id, detail:ali_parking_id, detail:ctime, detail:mtime") 
TBLPROPERTIES("hbase.table.name" = "hb_init_app_god_detail");

 

不懂欢迎咨询：Q:523225911