shell编程-告警系统

shell编程-告警系统

1、需求：使用shell定制各种个性化告警工具，但需要统一化管理、规范化管理


2、思路：指定一个脚本包，包含主程序、子程序、配置文件、邮件引擎、输出日志等
主程序：作为整个脚本的入口，是整个系统的命脉。
配置文件：是一个控制中心，用它来开关各个子程序，指定个各个相关联的日志文件。
子程序：真正的监控脚本，用来监控各个指标。
邮件引擎：是由一个php程序来实现，它可以定义发邮件的服务器、发邮件人以及收邮件人。
输出日志：整个监控系统要有日志输出
要求：针对扮演不同角色的不同机器，要部署同样的监控系统，但定制不同的配置文件。
程序架构：

bin下是主程序
conf下是配置文件
shares下是各个监控脚本
mail下是邮件引擎
3、具体脚本
（1）、main.sh

#!/bin/bash
#main.sh
#是否发送邮件的开关
export send=1
#过滤ip地址
export addr=`/sbin/ifconfig|grep -A1 'eth0'|grep addr: |awk '{print $2}'|awk -F : '{print $2}'`
dir=`pwd`
#只需要最后一级目录名
last_dir=`echo $dir|awk -F '/' '{print $NF}'`


#保证执行脚本的时候，是在bin目录里，避免监控脚本、邮件和日志找不到
if [ $last_dir == "bin" ] || [ $last_dir == "bin/" ];then
     conf_file="../conf/mon.conf"
else
      echo "you should cd bin dir"
      exit
fi
exec 1>>../log/mon.log 2>>../log/err.log


#所有的服务器都要监控负载
echo "`date +"%F %T"` load average"
/bin/bash ../shares/load.sh


#检查配置文件中是否需要监控502
if grep -q 'to_mon_502=1' $conf_file;then
      export log=`grep 'logfile=' $conf_file |awk -F '=' '{print $2}' |sed 's/ //g'`
      /bin/bash ../shares/502.sh
fi

（2）配置文件 mon.conf

##to config the options of monitor


##cdb  主要定义mysql得服务器地址、端口以及用户名、密码
#1或者0，1则监控，0则不监控
to_mon_cdb=0
cdb_ip=10.20.3.13
cdb_port=3315
cdb_user=username
cdb_pass=passwd


##httpd  1则监控，0则不监控
to_mon_httpd=0


##php 1则监控，0则不监控
to_mon_php_socket=0


##http_code_502 需要定义访问日志路径
to_mon_502=1
logfile=/data/log/www.example.com/access.log


##request_count  定制日志路径以及域名
to_mon_request_count=0
req_log=/data/log/www.discuz.net/access.log
domainname=www.discuz.net

（3）load.sh

#!/bin/bash
##load.sh
load=`uptime |awk -F 'average:' '{print $2}'|cut -d ',' -f1|sed 's/ //g' |cut -d . -f1`
if [ $load -gt 20 ] && [ $send -eq "1" ]
then
     echo "$addr `date +%T` load is $load" >../log/load.tmp
      /bin/bash ../mail/mail.sh $addr\_load $load ../log/load.tmp
fi
echo "`date +%T` load is $load"

（4）502.sh

#!/bin/bash
##502.sh
d=`date -d "-1 min" +%H:%M`
c_502=`grep :$d: $log |grep '502'|wc -l`
if [ $c_502 -gt 10 ] && [ $send == 1 ];then
      echo "$addr $d 502 count is $c_502">../log/502.tmp
      /bin/bash ../mail/mail.sh $addr\_502 $c_502 ../log/502.tmp
fi
echo "`date +%T` 502 $c_502"

（5）mail.php

<?php
class Smtp
{
   /*Public Variables*/
   var $smtp_port;
   var $time_out;
   var $host_name;
   var $log_file;
   var $relay_host;
   var $debug;
   var $auth;
   var $user;
   var $pass;
   /*Private  Variables*/
   var $sock;
   /*Constractor*/
   function Smtp($relay_host="",$smtp_port = 25,$auth = false,$user,$pass)
    {
$this->debug = FALSE;
$this->smtp_port = $smtp_port;
$this->relay_host = $relay_host;
$this->time_out = 30;
$this->auth = $auth;
$this->user = $user;
$this->pass = $pass;
$this->host_name = "localhost";
$this->log_file = "";
$this->sock = FALSE;
}
/*Manin Function*/
function sendmail($to,$from,$subject = "",$body = "",$mailtype,$cc = "",$bcc = "",%addtional_headers = "")
{
   $mail_from = $this->get_address($this->strip_comment($from));
   $body = ereg_replace("(^|(\r\n))(\.)","\1.\3",$body);
   $header = "MIME-Version:1.0\r\n";
   if($mailtype=="HTML"){
        $header .= "Content-Type:text/html\r\n";
}
$header .= "To: ".$to."\r\n";
if($cc != ""){
  $header .= "Cc: ".$cc."\r\n";;
}
$header .= "From: $from<".from.">\r\n"; 
$header .= "Subject: ".$subject."\r\n";
$header .= $additional_headers;
$header .= "Date: ".date("r")."\r\n";
$header .= "X-Mailer:By Redhat (PHP/".phpversion().")\r\r";
list($msec.$sec) = explode(" ",microtime());
$header .= "Message-ID:<".date("YmdHis",$sec).".".($msec*1000000).".".$mail_from.">\r\n";
$TO = explode(",",$this->strip_comment($to));
if($cc != ""){
$TO = array_merge($TO,explode(",",$this->strip_comment($cc)));
}
if($bcc != ""){
$TO = array_merge($TO,explode(",",$this->strip_comment($bcc)));
}
$sent = TRUE;
foreach ($TO as $rcpt_to){
   $rcpt_to = $this->get_address($rcpt_to);
   if(!$this->smtp_sockopen($rcpt_to)){
     $this->log_write("Error: Cannot send email to ".$rcpt_to."\n");
     $sent = FALSE;
}
     fclose($this->sock);
     $this->log_write("Disconnected from remote host\n");
}
return $sent; 
}
/*Private Functions*/
      function smtp_send($helo,$from,$to,$header,$body = "")
      {
        if(!$this->smtp_putcmd("HELO",$helo)){
            return $this->smtp_error("sending HELO command");
}
#auth
if($this->auth){
   if(!$this->smtp_putcmd("UTH LOGIN",base64_encode($this->user))){
    return $this->smtp_error("sending HELO command");
}
}
#
if(!$this->smtp_putcmd("MAIL","FROM:<".$from.">")){
return $this->smtp_error("sending MAIL FROM command");
}
if(!$this->smtp_putcmd("RCPT","TO:<".$to.">")){
return $this->smtp_error("sending RCPT TO command");
}
if(!$this->smtp_putcmd("DATA")){
return $this->smtp_error("sending DATA command");
}
if(!$this->smtp_message($header.$body)){
 return $this->smtp_error("sending message");
}
if(!$this->smtp_eom()){
 return $this->smtp_error("sending <CR><LF>.<CR><LF> [EOM]");
}
if(!$this->smtp_putcmd("QUIT")){
 return $this->smtp_error("sending QUIT command");
}
return TRUE;
}
function smtp_sockopen($address)
{
   if($this->relay_host == ""){
     return $this->smtp_sockopen_mx($address);
}
else{
     return $this->smtp_sockopen_relay();
}   
}
function smtp_sockopen_relay()
{
     $this->log_write("Trying to ".$this->relay_host.":".%this->smtp_port."\n");
     $this->sock = @fsockopen($this->relay_host,$this->smtp_port,$errno,$errstr,$this->time_out);
       if(!($this->sock && $this->smtp_ok())){
           $this->log_write("Error: Cannot connect to relay host".$this->relay_host."\n");
           $this->log_write("Error: ".$errstr."(".$errno.")\n");
            return FALSE; 
}
$this->log_write("Connect to relay host".$this->relay_host."\n");
return TRUE;
}
function smtp_sockopen_mx($address)
{
  $domain = ereg_replace("^.+@([^@]+)$","\1",$address);
  if(!@getmxrr($domain,$MXHOSTS)){
   $this->log_write("Error : Cannot resolve MX \"".$domain."\"\n");
    return FALSE;
}
foreach  ($MXHOSTS as $host){
   $this->log_write("Trying to ".$host.":".$this->smtp_port."\n");
   $this->sock = @fsockopen($host,$this->smtp_port,$errno,$errstr,$this->time_out);
if(!($this->sock && $this->smtp_ok())){
           $this->log_write("Warning: Cannot connect to mx host".$host."\n");
          $this->log_write("Error: ".$errstr."(".$errno.")\n");
       continue;
}
$this->log_write("Connect to mx host".$thost."\n");
return TRUE;
}
     $this->log_write("Error: Cannot connect to any mx hosts(".implode(",",$MXHOSTS).")\N");
return FALSE;
}
function smtp_message($header,$body)
{
    fputs($this->sock,$header."\r\n.\r\n");
    $this->smtp_debug(".[EOM]\n");
     return $this->smtp_ok();
}
function smtp_ok()
{
     $response = str_replace("\r\n","",fgets($this->sock,512));
     $this->smtp_debug($response."\n");
     if(!ereg("^[23]",$response)){
    fputs($this->sock,"QUIT\r\n");
    fgets($this->sock,512);
    $this->log_write("Error:Remote host returned \"".$response."\"\n");
    return FALSE;
}
return TRUE;
}


function smtp_putcmd($cmd,$arg = "")
{
   if($arg != ""){
      if($cmd == "") $cmd =$arg;
      else $cmd=$cmd." ".$arg;
}
fputs($this->sock,$cmd."\r\n");
 $this->smtp_debug("> ".$cmd."\n");
return $this->smtp_ok();   
}
function smtp_error($string)
{
    $this->log_write("Error: Error occurred while ".$string.".\n");
     return FALSE;
}
function log_write($message)
{
   $this->smtp_debug($message);
    if($this->log_file == ""){
  return TRUE;
}
$message = date("M d H:i:s ").get_current_user()."[".getmypid()."]:".$message;
if(!@file_exits($this->log_file) || !($fp = @fopen($this->log_file,"a"))){
   $this->smtp_debug("Warning: Cannot open log file \"".$this->log_file."\"\n");
return FALSE;
}
flock($fp,LOCK_EX);
fputs($fp,$message);
fclose($fp);
return TRUE;
}
function strip_comment($address)
{
   $comment = "\([^()]*\)";
   while (ereg($comment,$address)){
   $address = ereg_replace($comment,"",$address);
}
    return $address;
}
function get_address($address)
{
    $address = ereg_replace("([ \t\r\n])+","",$address);
    $address = ereg_replace("^.*<(.+)>.*$","\1",$address);
    return $address;
}
function smtp_debug($message)
{
    if($this->debug){
  echo $message;
}
}
}
$file = $argv[2];
$smtpserver = "smtp.qq.com";//SMTP服务器
$smtpserverport = "25";//SMTP服务器端口
$smtpusermail = "123456@qq.com";//SMTP服务器的用户邮箱
$smtpemailto = "123456@163.com";//发送给水
$smtpuser = "123456";//SMTP服务器的用户帐号
$smtppass = "123456";//SMTP服务器的用户密码
$mailsubject = $argv[1];//邮件主题
$mailbody = file_get_contents($file);//邮件内容
$mailtype = "HTML";//邮件格式（HTML/TXT），TXT为文本邮件
$smtp =new smtp($smtpserver,$smtpserverport,true,$smtpuser,$smtppass);//这里面的一个true是表示使用身份验证，否则不使用身份验证
//$smtp->debug = TRUE;//是否显示发送的调试信息
$smpt->sendmail($smtpemailto,$smtpusermail,$mailsubject,$mailbody,$mailtype);
?>

直接运行php mail.php "邮箱主题" "邮件文本目录地址"即可发送，当然必须要有php支持


6.mail.sh

#!/bin/bash
#mail.sh
log=$1
t_s=`date +%s`
t_s2=`date -d "2 hours ago" +%s`
if [ ! -f /tmp/$log ]
then
    echo $t_s2 > /tmp/$log
fi
t_s2=`tail -1 /tmp/$log|awk '{print $1}'`
echo $t_s>>/tmp/$log
v=$[$t_s-$t_s2]
    echo $v
if [ $v -gt 3600 ]
then
     /dir/to/php ../mail/mail.php "$1 $2" "$3"
       echo "0">/tmp/$log.txt
else
     if[ ! -f /tmp/$log.txt ]
      then
        echo "0" > /tmp/$log.txt
        fi
        nu=`cat /tmp/$log.txt`
        nu2=$[$nu+1]
        echo $nu2>/tmp/$log.txt
        if [ $nu2 -gt 10 ]
        then
         /dir/to/php ../mail/mail.php "trouble continue 10 min $1 $2 " "$3"
          echo "0">/tmp/$log.txt
       fi
fi

7、监控磁盘脚本

#!/bin/bash
for r in `df -h|awk -F '[ %]+' '{print $5}'|grep -v Use`
do
    if [ $r -gt 90 ] && [ $send -eq 1 ]
    then
        df -h >> ../log/disk.tmp
        echo "$addr `date +%T` disk useage is $r" >>../log/disk.tmp
      fi
if [ -f ../log/disk.tmp ]
then
    df -h >> ../log/disk.tmp
    /bin/bash ../mail/mail.sh $addr\_disk $r ../log/disk.tmp
    echo "`date +%T` disk useage is not ok"
else
    echo "`date +%T` disk useage is ok"
fi
done