11后续数据导入脚本

#! /bin/bash
#importdata.sh all/表名 日期
#1、判断参数是否传入
if [ $# -lt 1 ]
then
    echo "必须至少传入一个参数...."
    exit
fi
#2、获取日期[如果有传入日期用指定日期,如果没有传入日期用前一天的日期]
[ "$2" ] && datestr=$2 || datestr=$(date -d '-1 day' +%Y%m%d)

import_date(){
/opt/module/sqoop/bin/sqoop import \
--connect jdbc:mysql://hadoop102:3306/gmall \
--username root \
--password root123 \
--delete-target-dir \
--num-mappers 1 \
--split-by $2 \
--query "$1" \
--target-dir hdfs://hadoop102:8020/gmall/$3/$datestr \
--compress \
--compression-codec lzop \
--null-string '\\N' \
--null-non-string '\\N' \
--fields-terminated-by ,
#给lzo压缩文件创建索引
hadoop jar /opt/module/hadoop/share/hadoop/common/hadoop-lzo-0.4.20.jar com.hadoop.compression.lzo.DistributedLzoIndexer hdfs://hadoop102:8020/gmall/$3/$datestr
}
#3、根据参数匹配导入数据
case $1 in
#导入所有策略的表
"all")
import_date "select * from sku_info where \$CONDITIONS" "id" "sku_info"
import_date "select * from comment_info where DATE_FORMAT(create_time,'%Y%m%d')='$datestr' and \$CONDITIONS" "id" "comment_info"
import_date "select * from user_info where (DATE_FORMAT(create_time,'%Y%m%d')='$datestr' or DATE_FORMAT(operate_time,'%Y%m%d')='$datestr')and \$CONDITIONS" "id" "user_info"
;;
#导入指定表
#导入全量策略的表
"sku_info")
import_date "select * from sku_info where \$CONDITIONS" "id" "sku_info"
;;
#导入新增策略的表
"comment_info")
import_date "select * from comment_info where DATE_FORMAT(create_time,'%Y%m%d')='$datestr' and \$CONDITIONS" "id" "comment_info"
;;
#导入新增及修改策略
"user_info")
import_date "select * from user_info where (DATE_FORMAT(create_time,'%Y%m%d')='$datestr' or DATE_FORMAT(operate_time,'%Y%m%d')='$datestr')and \$CONDITIONS" "id" "user_info"
;;
*)
    echo "必须输入all或者是表名"
;;
esac