sqoop导入脚本——mysql_to_hdfs

#!/bin/bash

#操作命令格式
#sh mysql_to_hdfs.sh first 开始日期 结束日期（默认开始时间为前一天，结束时间为当天）
sqoop=/usr/hdp/3.1.4.0-315/sqoop/bin/sqoop
db=SAJT_DZFP_PT

do_date=`date -d '-1 day' +%F`
endDay=`date +%F`

if [[ -n "$2" ]]; then
    do_date=$2
fi

if [[ -n "$3" ]]; then
    endDay=$3
fi

while [ $do_date != $endDay ]
do
	import_data(){
	$sqoop import \
	--connect jdbc:mysql://xxxx:3306/$db \
	--username root \
	--password password \
	--target-dir /origin_data/test/$db/$1/$do_date \
	--delete-target-dir \
	--query "$2 and  \$CONDITIONS" \
	--num-mappers 1 \
	--fields-terminated-by '\t' \
	--compress \
	--compression-codec lzop \
	--null-string '\\N' \
	--null-non-string '\\N'

	hadoop jar /usr/hdp/3.1.4.0-315/hadoop/lib/hadoop-lzo-0.6.0.3.1.4.0-315.jar com.hadoop.compression.lzo.DistributedLzoIndexer /origin_data/test/$db/$1/$do_date
	}

	import_tbl_ec_document_item_1(){
	  import_data tbl_ec_document_item_1 "select
								 DIS_TAX,
								 CREATE_DATE
								from tbl_ec_document_item_1
								where date_format(CREATE_DATE,'%Y-%m-%d')='$do_date'"
	}
	import_tbl_ec_invoice_1(){
	  import_data tbl_ec_invoice_1 "select
	  							 CREATE_DATE,
								 EWM,
								 TZ_ERROR_MSG,
								 TZ_DATE
								from tbl_ec_invoice_1
								where date_format(CREATE_DATE,'%Y-%m-%d')='$do_date'"
	}
	
	case $1 in
	"tbl_ec_document_item_1")
	  import_tbl_ec_document_item_1
	;;
	"tbl_ec_invoice_1")
	  import_tbl_ec_invoice_1
	;;
	"first")
	  import_tbl_ec_document_item_1
	  import_tbl_ec_invoice_1
	esac
	do_date=`date -d "+1 day ${do_date}" +%F` #获取第二天的时间
done