Curl 工具的妙用：shell中用curl实现业务监控和故障自动处理

不时会遇到JVM内存溢出或服务进程僵死导致的应用不可访问的情况，这个时候一般需要重启下服务进程。虽然在zabbix中对web的监控能监控到且可以执行远程命令，但是基于安全考虑，一般不会再zabbix 的agent上开启允许执行远程命令的权限。可以在本机安装curl工具来实现对web的监控，辅助些脚本就可以实现故障和恢复的告警发送和进程重启了。

#!/usr/bin/bash
##############################################
#NAME: monweb.sh 
#DESC:mon the web status,if wrong restart container.
#Note: Curl must be installed，curl工具必须提前安装好。
#History:
#   v1.0 2016-1-20 thinker yang
##############################################
#变量说明，调试级别，数字越高输出信息越多
DEBUG=2
#
#define config
#
##local conf##
#需要载入运行中间件用户下的profile文件，因为启停脚本的可能部分变量定义在此
. /.profile
#定义时间格式
TIME=`date`
#定义要监控页面的curl
URL=wwww.baidu.com/123.html
#定义短信告警头
AlertHead=ERROR-TOMCAT
##server conf##
##中间件相关配置##
#定义启动中间件的命令脚本
STARTSERVERCMD=/startapp.sh
#定义正常停止中间件的命令脚本
STOPSERVERCMD=/stopserver.sh
#用以下关键字grep出中间件进程的pid，如果用一个关键字就可以过滤出，则后面两个和第一个一致即可
PSKEY1=java
PSKEY2=java
PSKEY3=java
#sleep时间/秒：完全停止中间件后到再次启动中间件等待的秒数
WAITRESTARTTIME=10
#
#sleep时间/秒：中间件重启完成后，用curl再次检测web页面状态的间隔时间
WAITCHECKHEALTHTIME=60
##java core file conf##
#内存溢出后产生java dump 文件的路径
MEMOUTFILEPATH=/tmp
#用grep关键字查找因内存溢出而产生文件名的关键字（一般不需要改）
MEMOUTFILEKEY1=heapdump
MEMOUTFILEKEY2=phd
#是（Y）否（N）删除这些因内存溢出产生的dump core文件，默认为删除
RMJAVACOREFILEFLAG=Y
##log conf##
#脚本输出日志路径和日志文件名配置
LOGDIR=/tmp/monlog
LOG=$LOGDIR/monweb.sh.log
##alert sms config##

#########################
#
#function:sendsms,此处定义告警信息来实现邮件或者短信告警，使用方式为echosm "告警内容"，需要自己编写。
#   如果不定义，直接定义为echo $1 就可以
#para:@message
#
#########################
echosm()
{
    echo $1
}
#########################
#
#function:getHttpStatus
#
#########################
getHttpStatus()
{
    HTTPCODE=`curl -o /dev/null -s -w %{http_code} $URL`
    return $HTTPCODE
}
#########################
#
#function:getHttpStatus
#return: container's process number.
#
#########################
getServerProcess()
{
    test $DEBUG -ge 1 && echo "DEBUG1> IN function getServerProcess echo PSKEY1: $PSKEY1"
    test $DEBUG -ge 1 && echo "DEBUG1> IN function getServerProcess echo PSKEY2: $PSKEY2"
    test $DEBUG -ge 1 && echo "DEBUG1> IN function getServerProcess echo PSKEY3:$PSKEY3"
    serverProcessNum=`ps -ef | grep -v grep | grep $PSKEY1 | grep $PSKEY2 | grep $PSKEY3 | wc -l`
    return $serverProcessNum
    test $DEBUG -ge 1 && echo "DEBUG1> function getServerProcess return serverProcessNum $serverProcessNum"
}
#########################
#
#function:killServer,Frist use normal cmd stop server,if fail then kill.
#
#########################
killServer()
{
    test $DEBUG -ge 1 && echo "DEBUG1> Func killServer Begin!" 
    $STOPSERVERCMD
    sleep $WAITRESTARTTIME
    getServerProcess
    #If staill has apache process，then killed
    if [ $serverProcessNum -gt 0 ];then
        ps -ef | grep -v grep | grep $PSKEY1 | grep $PSKEY2 | grep $PSKEY3 | awk '{print $2}'|xargs kill -9
    fi
}
#########################
#
#function:startServer,normal start container server.
#
#########################
startServer()
{
    $STARTSERVERCMD
}
#########################
#
#function:restartServer
#
#########################
restartServer()
{
    killServer
    sleep $WAITRESTARTTIME
    startServer
    echo "$AlertHead has been restarted! at $TIME on `hostname`" | tee -a  $LOG
    echosm "$AlertHead has been restarted! at $TIME on `hostname`" | tee -a  $LOG
}
#########################
#
#function:getOutMemFileNum,when Java memory out ,it will be dump some core files.
#return: dump core files number
#
#########################
getOutMemFileNum()
{
    if [ ! -d $MEMOUTFILEPATH ];then
        echo "$MEMOUTFILEPATH not exist,please check again!"
    else
        if [ ${RMJAVACOREFILEFLAG} = "Y" ];then
            heapdumpNum=`ls $MEMOUTFILEPATH | grep $MEMOUTFILEKEY1 | grep $MEMOUTFILEKEY2 | wc -l`
            return $heapdumpNum
        else
            heapdumpNum=0
            return $heapdumpNum
        fi
    fi
}
#########################
#
#function:delMemOutFile,Those Java memory dump core files very too larger,should be delete.
#
#########################
delMemOutFile()
{
    cd $MEMOUTFILEPATH
    ls $MEMOUTFILEPATH | grep $MEMOUTFILEKEY1 | grep $MEMOUTFILEKEY2 | xargs /usr/bin/rm -f
}
#########################
#
#function:main
#
#########################
main()
{
    #test ! -d $LOGDIR && mkdir -p $LOGDIR
    if [ ! -d $LOGDIR ];then
        mkdir -p $LOGDIR
    fi
    getOutMemFileNum
    getHttpStatus
    if [ "$HTTPCODE"x != "200"x ] || [ $heapdumpNum -ge 1 ];then
            echo "$AlertHead httpcode is $HTTPCODE ! at $TIME on `hostname` " | tee -a  $LOG
            #If exist apache process but httpcode is not 200 it will be restart,if there is no apache process then no work on it.
            getServerProcess
            test $DEBUG -ge 1 && echo "Frist run getServerProcess in func main return serverProcessNum: $serverProcessNum "
            if [ $serverProcessNum -ge 1 ];then
                echosm "$AlertHead httpcode is $HTTPCODE and will be restarted ! at $TIME on `hostname` " | tee -a  $LOG
            fi
            test $DEBUG -ge 2 && test $serverProcessNum -ge 1 && echo "$AlertHead httpcode is $HTTPCODE and will be restarted ! at $TIME on `hostname` " | tee -a  $LOG
            test $DEBUG -ge 1 && echo "Second run getServerProcess in func main return serverProcessNum: ${serverProcessNum} ,before restart!"
            getServerProcess
            if [ $serverProcessNum -ge 1 ];then 
                restartServer
            fi
            test $DEBUG -ge 2 && echo "DEBUG2> After restartServer the serverProcessNum : ${serverProcessNum} "
            sleep $WAITCHECKHEALTHTIME
            getHttpStatus
            echo "$AlertHead status is $HTTPCODE, at $TIME on `hostname` ." | tee -a  $LOG
            if [ "$HTTPCODE" = "200" ];then
                echosm "$AlertHead has OK now!at $TIME on `hostname` " | tee -a  $LOG
            else
                echosm "$AlertHead restart failed or after restart the web staill wrong,please check !at $TIME on `hostname` " | tee -a  $LOG
            fi
    else
            echo "$AlertHead status is $HTTPCODE, at $TIME on `hostname` ." | tee -a $LOG
    fi
    if [ ${RMJAVACOREFILEFLAG} = "Y" ];then
        delMemOutFile
        if [ $? -eq 0 ];then
            echo "JAVA Core Files in $MEMOUTFILEPATH has been rm, at $TIME ON `hostname` . " | tee -a  $LOG
        fi
    else
        echo "JAVA Core Files in $MEMOUTFILEPATH staill exist,Please check you free space on filesystem at $TIME ON `hostname` . " | tee -a  $LOG
    fi
}
#
#MAIN EXE
#
main