Prometheus告警到钉钉群

说明：
这是第一版代码初步测试功能上没有问题，完成于2022年5月。
版本3更新于2023年3月20日，主要添加http异常处理和日志输出功能，目前仍在更新中

import requests,json,time,hmac,hashlib,base64,urllib.parse,time
# from random import randint,choice
# from string import digits,ascii_letters
from hashlib import md5

class Monitor:
    def __init__(self):
        # prometheus地址
        self.usr = "http://192.168.10.36:9090/"

    def send_msg(self,webhook_url, secret, remiders, msg):
        headers = {'Content-Type': 'application/json; charset=utf-8'}

        # 生成当前时间戳，单位是毫秒，与请求调用时间误差不能超过1小时
        timestamp = str(round(time.time() * 1000))
        # 修改编码格式为utf-8
        secret_enc = secret.encode('utf-8')
        # 将timestamp和secret合并
        string_to_sign = '{}\n{}'.format(timestamp, secret)
        # 修改编码格式为utf-8
        string_to_sign_enc = string_to_sign.encode('utf-8')
        # 将字段进行加密，加密类型采用sha256
        hmac_code = hmac.new(secret_enc, string_to_sign_enc, digestmod=hashlib.sha256).digest()
        sign = urllib.parse.quote_plus(base64.b64encode(hmac_code))
        # 生成请求的URL，WebHook地址
        Post_Url = ("%s&timestamp=%s&sign=%s" % (webhook_url, timestamp, sign))
        # return Post_Url
        data = {
            "msgtype": "text",
            "at": {
                "atMobiles": remiders,
                "isAtAll": False,
            },
            "text": {
                "content": msg,
            }
        }
        r = requests.post(Post_Url, data=json.dumps(data), headers=headers)
        return r.text

    'random_keys 弃用'
    # def random_keys(self):
    #     keys = ''
    #     for i in range(10):
    #         keys += choice(ascii_letters)
    #     return keys

    def alert(self):
        """
        获取告警列表
        :return:
        """
        print("this is prometheus alerts")
        # url = self.usr + 'api/v1/rules?type=alert'
        url = self.usr + '/api/v1/alerts'

        response = requests.request('GET', url)
        if response.status_code == 200:
            # rules = response.json()['data']['groups'][0]['rules']
            alerts = response.json()['data']['alerts']

            # print(alerts)
            # print(len(alerts))
            # if len(alerts) == 0:
            #     return 0

            alerts_data=[]
            for alert in alerts:
                # value = '%2.2f' % float(alert['annotations']['value'])
                # node = alert['labels']['instance'].split(":")[0]
                node = alert['labels']['instance']
                alert_time=time.strftime('%Y-%m-%d %H:%M:%S')
                md5name = alert['labels']['alertname']+node
                mvalue=md5(md5name.encode("utf8")).hexdigest()[8:24]

                try:
                    value = '%2.2f' % float(alert['annotations']['value'])
                except KeyError:
                    value = 0

                data = {
                    "告警标题:":alert['labels']['alertname'],
                    "告警节点:":node,
                    "告警级别:":alert['labels']['severity'],
                    "告警详情:":alert['annotations']['description'],
                    "触发时间:":alert_time,
                    "触发值:": value,
                    '校验值:' : mvalue
                }
                keys = mvalue
                random_dist = {keys:data}
                alerts_data.append(random_dist)
            return alerts_data
        else:
            print('Get targets status failed!')
            print("############ Monitor ending ####################")
            return None

remiders = [15038***605]
webhook_url = "https://oapi.dingtalk.com/robot/send?access_token=916e3d3d781704e24ee77b09d66b3eeee*****"
secret = "SEC8fd587da934****"

akeys = []
akeys2 = []
alist = []
alist2 = []

class prometheus_alerts:
    def __init__(self):

        self.cm=Monitor()
        self.alerts = self.cm.alert()
        global akeys,akeys2,alist,alist2

    def status(self):
        if len(self.alerts) == 0:
        # if self.alerts == 0:
            # print("self.alerts() == 0")
            return 0

    def alarm(self):
        # self.mon_values = {}

        # print("########## alarm starting ##########")
        if self.status() == 0:
            print("self.status() == 0")
            # print("########## alarm ending ##########")
            return

        for line in self.alerts:
            # print("self.status != 0")
            # print("line: ",line)

            for key,values in line.items():
                '此处应添加md5值判断'
                # print("校验值:",values['校验值:'])

                '取出 alerts中值 转换成md5'
                md5name2=values['告警标题:']+values['告警节点:']
                mvalues2=md5(md5name2.encode("utf8")).hexdigest()[8:24]

                if values['校验值:'] not in akeys:
                    akeys.append(key)
                    alist.append(line)

                # print("alarm-akeys: ",akeys)
                # print("alarm-alist: ",alist)
                msg = ''
                '打印告警信息'
                for value in values:
                    if value == '校验值:':
                        continue
                    msg += value + " " + str(values[value]) + "\n"
                self.cm.send_msg(webhook_url,secret,remiders,msg)
                # print(msg)

    def recovery1(self):
        '开始无告警，之后无告警'
        global akeys,alist
        if self.status() == 0:
            # print("self.status() == 0")
            if not akeys:
                return

        # '假设 所有告警都恢复'
            adict = {}
            for dict_value in alist:
                adict.update(dict_value)

            if akeys:
                for keys,values in adict.items():
                    msg = ''
                    for name in values:
                        if name == '告警级别:':
                            values[name] = '恢复正常'
                        if name == '校验值:':
                            continue
                        msg += name + " " + str(values[name]) + "\n"
                    self.cm.send_msg(webhook_url, secret, remiders, msg)
                    print(msg)

                alist = []
                akeys = []

        'self.status() != 0'
        '假设 恢复部分告警'
        # print()
        # if self.status() != 0:
        bkeys=[] # 临时存放alerts中的 key

        for line in self.alerts:
            for key,values in line.items():
                bkeys.append(key)

        adict = {}

        for dict_value in alist:
            adict.update(dict_value)

        adict2 = adict.copy()
        akeys2 = akeys.copy()

        # print("adict2:",adict2)
        print("akeys2:",akeys2)
        for i in akeys:    # '判断akeys值是否存在alerts中,不存在则告警恢复'
            print("akeys: ",i)
            if i not in bkeys:
                '4、打印恢复告警信息'
                msg = ''
                for name in adict[i]:
                    if name == "告警级别:":
                        adict[i][name] = "恢复正常"
                    if name == '校验值:':
                        continue
                    msg += name + " " + str(adict[i][name]) + "\n"
                self.cm.send_msg(webhook_url, secret, remiders, msg)
                # print(msg)

                '5、删除akeys和alist中已恢复告警的相关信息'
                akeys2.remove(i)
                adict2.pop(i)

        akeys = akeys2.copy()

        alist = []
        for key,values in adict2.items():
            alert_list = {key:values}
            alist.append(alert_list)

    def control(self):
        ''
        self.alarm()
        self.recovery1()

if __name__ == '__main__':
    while True:
        pa = prometheus_alerts()
        pa.alarm()
        for i in range(6):
            # print(i)
            # pa = prometheus_alerts()
            pa.recovery1()
            time.sleep(30)