背景
公司一般做业务监控都是采用实时任务的方式,实时任务的可用性就更加需要保障了。
监控脚本
#!/bin/bash
#去azkaban数据库获取所有的实时任务名称,stream开头的
#stream.sql
#use azkaban;
#SELECT name FROM azkaban.projects where name like 'stream_%' and active =1;
mysql -hhdh06.c.p.xyidc -uuser -ppasswd <stream.sql >a.txt
yarn application -list |grep stream >b.txt
sed -n '2,$p' a.txt|while read line
do
name=`sed -n /$line/p b.txt `
if [ "$name" != "" ];then
appid=`sed -n /$line/p b.txt |awk '{print $1}'`
url=`sed -n /$line/p b.txt |awk '{print $9}'`
echo $line -- $appid
$(curl -L --connect-timeout 20 ${url}/streaming >c.txt)
$(sed -i 's/})/a/g' c.txt)
active=`sed -n '/Active Batches/p' c.txt |awk -F"[()]" '{print $2}'`
echo active:$active
if [ -z $active ] ;then
echo 实时任务${line}运行异常,任务id:${appid},不能获取driver运行状态,快检查driver日志!
curl "http://xxxx:8080/alarm/sendSms.do?mobile=15158137***&type=0&producer=CDH&body=实时任务告警,实时任务${line}运行异常,任务id:${appid},不能获取driver运行状态,快检查driver日志!"
elif [ $active -gt $1 ];then
echo 实时任务${line}运行出现堆积,任务id:${appid},当前堆积${active}批次!
curl "http://xxxx:8080/alarm/sendSms.do?mobile=1515813***&type=0&producer=CDH&body=实时任务告警,实时任务${line}运行出现堆积,任务id:${appid},当前堆积${active}批次!"
else
echo $appid 任务运行正常!
fi
else
echo 实时任务告警,未检测到运行任务,实时任务${line}已停止运行!请检查任务日志!
curl "http://xxxx:8080/alarm/sendSms.do?mobile=1515813***&type=0&producer=CDH&body=实时任务告警,未检测到运行任务,实时任务${line}已停止运行!请检查任务日志!"
fi
done