加群:397706991,共同学习
conf/hive.xml
具体Shell代码下载:http://download.csdn.net/detail/luo849278597/9490920
<?xml version="1.0" encoding="UTF-8"?>
<root>
<hive-shell type="analysis"><!-- 数据分析 -->
<hsql>create table if not exists think_statistics(date_type_name string,date_name string,type int,type_name string,count int) row format delimited fields terminated by '\\t' lines terminated by '\\n' stored as textfile</hsql>
<hsql>insert into think_statistics select concat(current_date,'-2'),current_date,2,'新任务数量',sum(case when status==2 then 1 else 0 end) from think_task</hsql><!-- hive 查询语句 -->
</hive-shell>
</root>
bin/hive.sh
#!/bin/bash
########################
#以下命令的处理基于Hadoop环境以及Hive工具,并且相应的工具命令需要定义在环境变量中
#sed命令是Shell编程中用于处理字符串,过滤出所需字符串的命令
#tr -d '\r' 用于删除每行中的换行符
#eval 用于执行字符串命令
########################
#get the environment
if [ -f ~/.bashrc ];#进入Bash Shell命令,用于以下命令的执行
then
. ~/.bashrc
fi
#file exit
if [ -z $1 ] ; then #判断配置文件是否存在
echo 'USAGE:COMMAND FILENAME'
exit 0
fi
#预定义变量以及数组
declare -a analysis_sqls
declare -i i=0
FLAG=0
content=`sed -e 's/\s*\(.*\)\s*$/\1/g' -e 's/\s*\(=\)\s*/\1/g' -e '/^\(\s\)*$/d' -e '/^$/d' -e 's/<!--.*-->//' $1`
while read line
do
if echo ${line}|grep -qE "^<hive-shell type=\"analysis\"" ; then
FLAG=1
continue
fi
if echo ${line}|grep -qE "</hive-shell>" ;then
FLAG=0
continue
fi
if [ ${FLAG} -ne 0 ] ; then
sql=`echo ${line}|sed -n -e 's/<hsql>//' -e 's/<\/hsql>//p'|tr -d '\r'`
fi
#empty sql,then wo will not excute follow
if [ ${#sql} -eq 0 ] ; then
continue
fi
if [ ${FLAG} -eq 1 ] ; then
analysis_sqls[$i]=$sql
let i++
continue
fi
done<<EOF
$content
EOF
shellsql="echo -e \"\033[41:33m excute hive.sh start analysis sqls \033[0m\""
for ((i=0;i<${#analysis_sqls[*]};i++))
do
shellsql="${shellsql} && hive -e \"${analysis_sqls[$i]}\""
done
shellsql="${shellsql} && echo -e \"\033[41:33m excute hive.sh finished analysis sqls \033[0m\""
eval $shellsql