生产环境下Sqoop配置文件

sqoop生产环境配置-全量拉取pgsql数据库

#! /bin/bash

sqoop=/opt/module/sqoop/bin/sqoop

if [ -n "$2" ] ;then
    do_date=$2
else
    do_date=`date -d '-1 day' +%F`
fi

import_data(){
$sqoop import -D mapreduce.map.cpu.vcores=2 -D mapreduce.map.memory.mb=4096 -D mapreduce.map.java.opts=-Xmx8192m -D mapreduce.task.io.sort.mb=1024 -D mapreduce.task.timeout=3600000 \
--connect $1 \
--username $2 \
--password $3 \
--target-dir /origin_data/robot/db/$4/$do_date \
--delete-target-dir \
--query "$5 and  \$CONDITIONS" \
--fields-terminated-by '\001' \
--hive-drop-import-delims \
--split-by "$6" \
--num-mappers 4 \
--compress \
--compression-codec lzop \
--null-string '\\N' \
--null-non-string '\\N' \
-- --schema "$7"

hadoop jar /opt/module/hadoop-3.1.3/share/hadoop/common/hadoop-lzo-0.4.20.jar com.hadoop.compression.lzo.DistributedLzoIndexer /origin_data/robot/db/$4/$do_date
}
##售后 全量表
import_inventory_stockmoverecord_materials(){
  import_data jdbc:postgresql://pgr-************.aliyuncs.com:1921/smarterp 账号 密码 inventory_stockmoverecord_materials "select
                          id,
                          stockmoverecord_id,
                          material_id
                        from smarterp.inventory_stockmoverecord_materials where (1 = 1)" id smarterp
}


case $1 in
##售后
  "inventory_stockmoverecord_materials")
     import_inventory_stockmoverecord_materials
;;
  "all")
##售后
    import_inventory_stockmoverecord_materials
;;
esac