Linux下Shell编程实现基于Hadoop的ETL(分析篇)

加群：397706991，共同学习

conf/hive.xml

具体Shell代码下载:http://download.csdn.net/detail/luo849278597/9490920

<?xml version="1.0" encoding="UTF-8"?>
<root>
	<hive-shell type="analysis"><!-- 数据分析 -->
		<hsql>create table if not exists think_statistics(date_type_name string,date_name string,type int,type_name string,count int) row format delimited fields terminated by '\\t' lines terminated by '\\n' stored as textfile</hsql>
		<hsql>insert into think_statistics select concat(current_date,'-2'),current_date,2,'新任务数量',sum(case when status==2 then 1 else 0 end) from think_task</hsql><!-- hive 查询语句 -->
	</hive-shell>
</root>

bin/hive.sh

#!/bin/bash
########################
#以下命令的处理基于Hadoop环境以及Hive工具，并且相应的工具命令需要定义在环境变量中
#sed命令是Shell编程中用于处理字符串，过滤出所需字符串的命令
#tr -d '\r' 用于删除每行中的换行符
#eval 用于执行字符串命令
########################
#get the environment
if [ -f ~/.bashrc ];#进入Bash Shell命令，用于以下命令的执行
then
 . ~/.bashrc
fi
#file exit
if [ -z $1 ] ; then #判断配置文件是否存在
	echo 'USAGE:COMMAND FILENAME'
	exit 0
fi

#预定义变量以及数组
declare -a analysis_sqls
declare -i i=0
FLAG=0
content=`sed -e 's/\s*\(.*\)\s*$/\1/g' -e 's/\s*\(=\)\s*/\1/g' -e '/^\(\s\)*$/d' -e '/^$/d' -e 's/<!--.*-->//' $1`

while read line
do

	if echo ${line}|grep -qE "^<hive-shell type=\"analysis\"" ; then
		FLAG=1
		continue
	fi

	if echo ${line}|grep -qE "</hive-shell>" ;then
		FLAG=0
		continue
	fi

	if [ ${FLAG} -ne 0 ] ; then
		sql=`echo ${line}|sed -n -e 's/<hsql>//' -e 's/<\/hsql>//p'|tr -d '\r'`
	fi
	#empty sql,then wo will not excute follow
	if [ ${#sql} -eq 0 ] ; then
		continue
	fi

	if [ ${FLAG} -eq 1 ] ; then
		analysis_sqls[$i]=$sql
		let i++
		continue
	fi
done<<EOF
$content
EOF

shellsql="echo -e \"\033[41:33m excute hive.sh start analysis sqls \033[0m\""
for ((i=0;i<${#analysis_sqls[*]};i++))
do
	shellsql="${shellsql} && hive -e \"${analysis_sqls[$i]}\""
done
shellsql="${shellsql} && echo -e \"\033[41:33m excute hive.sh finished analysis sqls \033[0m\""
eval $shellsql