#!/bin/bash
tab_nm=$1
etldt=`date -d $2 + "%Y%m%d"`
dt=`date -d $2 + "%Y-%m-%d"`
echo "${tab_nm}_${dt}"
#数据存放目录
data_hdfs_dir ="/tmp/idap/export_data/${tab_nm}"
#数据导出路経
local_path ="/gpfsetl/jcp_dw/script/idap/test/${tab_nm}_${dt}"
#scp将数据传到目标服务器上
scp_cmd ="user@xxx.xxx.xx.xxx:/home/dsadm/ycy/export_data"
#删除历史数据
rm -rf ${local_path}
if [ $? -ne 0 ]; then
echo "rm -rf ${local_path} ERROR !"
exit 2
else
echo "删除1ocal数据成功!"
fi
export HADOOP_USER_NAME = dsadm
hadoop fs -rm -r -f ${data_hdfs_dir}
if [ $? -ne 0 ]; then
echo " hadoop fs -rm -r -f ${data_hdfs_dir} ERROR !"
exit 2
else
echo "删除 hdfs 数据成功!"
fi
#创趣 hdfs 目录
echo "创建 hdfs 目录:${data_hdfs_dir}"
hadoop fs -mkdir -ps ${data_hdfs_dir}
if [ $? -ne 0 ] ; then
echo " hadoop fs -mkdir -p ${local_path} ERROR !"
exit
e1se
echo "创建 hdfs 日录:${local_path} 成功!"
fi
echo "创建1ocal目录: ${local_path}"
mkdir ${local_path}
if [ $? -ne 0 ]; then
echo " hadoop fs -mkdir -p ${local_path} ERROR!"
exit 2
else
echo "创建1ocal目录: ${local_path}成功!"
fi
hadoop fs -chmod -R 777 ${data_hdfs_dir}
#导出语句
export_sql="select * from idap.${tab_nm} where last_etl_acg_dt = '${dt}' "
echo "--将数据导出到 HDFS 上--"
beeline -u jdbc:hive2://olap14:10000 -e
"set mapred.reduce.tasks=-1; insert overwrite directory '${data_hdfs_dir}'
row format serde 'org.apache.hadoop.hive.serde2.OpenCSVSerde' ${export_sql} distribute by 1;"
if [ $? -ne 0 ]; then
echo "将数据导出到 HDFS 上 failed !"
exit 2
else
echo "将数据导出到 HDFS 上 success !"
echo "--将数据从 HDFS 导出到本地------"
fi
hadoop fs -getmerge ${data_hdfs_dir} ${local_path}/${tab_nm}_${etldt}.del
if [ $? -ne 0 ]; then
echo "将数据从 HDFS 导出到本地失败!"
exit 2
else
echo "特数据从 HDFS 导出到本地成功!"
fi
# Scp 将数据传到目标服务器上
scp ${local_path}/${tab_nm}_${etldt}.del ${scp_cmd}
if [ $? -ne 0 ]; then
echo "scp 将数据传到目标服务器上失败!"
exit 2
else
echo "scp 将数据传到目标服务器上成功!"
fi
#删除临时数据文件
echo "----删除临时数据文件----"
hadoop fs -rm -r -f ${data_hdfs_dir}
rm -rf ${local_path}
hive数据导出成csv文件shell脚本
于 2022-04-06 09:50:29 首次发布