HDFS 下SHELL脚本监控表信息

SHELL脚本监控表信息，触发电话或者其他服务

 

#!/bin/sh
source /etc/profile
sleep 10

echo "v20180202"

data_date=`date -d "$today" +%Y%m%d --date="-1 day"`

function getPath()
{
 sql_text="select s.LOCATION
from dbs d
inner join tbls t on d.DB_ID = t.DB_ID
inner join partitions p on t.TBL_ID = p.TBL_ID
inner join sds s on p.SD_ID = s.SD_ID
where d.name='${1}'
and t.TBL_NAME='${2}'
and p.PART_NAME='${3}'
;"

 hdfs=`mysql -hsfbdp1mtstore-m.db.sfdc.com.cn -ucheck_user -psf123456 -Dsfbdp1mtstore -s -N -f -e "${sql_text}"`
 echo ${hdfs}
}

function check_hdfs()
{
 #通过元数据获取hdfs路径
 fullPath=`getPath "$1" "$2" "$3"`

 #检测分区是否已创建
 if [ ! $fullPath ]; then
   echo "#表$1.$2分区($3)不存在。"
  return 1
 fi

 #分区已创建的情况下 检测hdfs目录是否存在
 hadoop fs -test -d $fullPath
 if [ $? -ne 0 ] ;then
  echo "#"$fullPath" 不存在。"
  return 2
 fi

 #hdfs目录存在的情况下 检测hdfs大小
 fileSize=`hadoop fs -du $fullPath | awk '{sum += $1};END {print sum}'`
 if [ ! $fileSize ]; then
  fileSize=0
 fi
 echo "#HDFS详细路径:"$fullPath
 echo "#占用存储:"$fileSize
 if [ $fileSize -lt 10 ]; then
  echo "#HDFS详细路径:"$fullPath "--占用存储:"$fileSize
  return 3
 fi

 return 0
}

tablename_array=( 
"nmc_cost_item_cost_sts_day" 
"nmc_cost_cust_batch_dtl"
"nmc_cost_buss_overview_sts_day" 
"nmc_buss_overview_sts"
"nmc_incm_overview_progress_year" "nmc_incm_overview_progress_mon" "nmc_incm_prod_strct_year" "nmc_incm_prod_strct_mon"
"nmc_incm_cust_type_overview_year" "nmc_incm_cust_type_overview_mon" "nmc_incm_cllct_serv_overview_year" "nmc_incm_cllct_serv_overview_mon"
"nmc_incm_ce_overview_year" "nmc_incm_ce_overview_mon" "nmc_incm_item_1level_trend_mon" "nmc_incm_item_1level_trend_year" "nmc_incm_buss_sts_year"
"nmc_incm_buss_sts_mon" "nmc_incm_prod_main_prod_sts_year" "nmc_incm_prod_main_prod_sts_mon" "nmc_incm_prod_main_prod_cust_sts_year" "nmc_incm_prod_main_prod_cust_sts_mon"
"nmc_incm_cust_add_value_sts_year" "nmc_incm_cust_add_value_sts_mon" "nmc_incm_ce_prj_org_buss_sts_mon"
"nmc_incm_ce_prj_org_buss_sts_year" "nmc_incm_emp_buss_sts_mon" "nmc_incm_emp_buss_sts_year" "nmc_incm_cust_prj_incm_sts_mon" "nmc_incm_cust_prj_incm_sts_year" "nmc_incm_cust_cod_incm_sts_mon"
"nmc_incm_cust_cod_incm_sts_year" "nmc_incm_cust_mid_high_buss_sts_mon"
"nmc_incm_cust_buss_trend_mon" "nmc_incm_mh_cust_ana_mon" "nmc_incm_cust_chnl_ana_mon")
emp_code_list="01374213,01384134,01367683"
v_count=-1
f_total_count=0
s_total_count=0
while :
do
 v_count=$(($v_count+1))
 tablename=${tablename_array[$v_count]}
 echo "================================================"$tablename

 if [ ! -n "$tablename" ]; then
   echo "任务已完成"
   exit 0
 else
   echo "NOT NULL"
 fi

 check_hdfs "dm_nmc" $tablename "inc_day="${data_date}

 #不等于0时说明数据未准备好
 if [ $? -ne 0 ]; then
   f_total_count=$(($f_total_count+1))
   curl -l -H "Content-type: application/json" -X POST -d "{\"userId\": \"${emp_code_list}\",\"alertMode\":\"1,2,3\",\"content\":\"ERR数据异常，多谢！${tablename} inc_day=${data_date}分区为0 此信息为当前检测到的第 ${f_total_count} 个问题表\",\"subject\":\"XXXXX项目\"}" http://bdp-falcon-proxy.sf-express.com/falcon/notice/send 
   else
   echo "正常"
 fi

done