python3+elasticsearch实现接口数据检查发送至企业微信

前言：萌新一枚，按需参考，与君共勉

功能：通过python获取elasticsearch的数据，做接口统计巡检

代码如下：

import requests
import json
import datetime
import sys
from elasticsearch import Elasticsearch                  #elasticsearch模块需和elasticsearch大版本一致
es_index = 'ingress_*'         ##搜索的索引名
es_hosts = "efk-elasticsearch-coordinating-only.cloud-infra.svc.cluster.local"        ##主机地址，样例为容器地址

now = (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
five_minute_age = (datetime.datetime.now() - datetime.timedelta(hours=24)).strftime('%Y-%m-%d %H:%M:%S')


def send_msg(msg):
    alert_time_rage = "统计时间范围:" +five_minute_age + ' ~ ' + now + "\n"
    title_name = "## 接口巡检\n"
    url = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=693axxx6-7aoc-4bc4-97a0-0ec2sifa5aaa"
    headers = {
        "Content-Type": "application/json"
    }
    data = {
        "msgtype": "markdown",
        "markdown": {
            "content": title_name + alert_time_rage + msg
            }
    }
    r = requests.post(url, headers=headers, json=data)


def get_count():
    es = Elasticsearch(hosts=es_hosts, port=9200, timeout=200)          ##http_auth=(es_user, es_pwd)有用户名加认证进去，
    query =  {
  "aggs": {
    "max_count": {
      "terms": {
        "field": "request.keyword",
        "order": {
          "_count": "desc"
        },
        "size": 10
      }
    }
  },
  "size": 0,
  "_source": {
    "excludes": []
  },
  "stored_fields": [
    "*"
  ],
  "script_fields": {},
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    }
  ],
  "query": {
    "bool": {
      "must": [],
      "filter": [
        {
          "match_all": {}
        },
        {
          "exists": {
            "field": "query"
          }
        },
        {
          "bool": {
            "should": [
              {
                "match_phrase": {
                  "request": "api"
                }
              }
            ],
            "minimum_should_match": 1
          }
        },
        {
          "range": {
            "@timestamp": {
              "format": "strict_date_optional_time",
              "gte": "now-24h",
              "lte": "now"
            }
          }
        }
      ],
      "should": [],
      "must_not": [
        {
          "match_phrase": {
            "status": {
              "query": "302"
            }
          }
        },
        {
          "bool": {
            "should": [
              {
                "match_phrase": {
                  "request": "grafana"
                }
              }
            ],
            "minimum_should_match": 1
          }
        }
      ]
    }
  }
}

    
    result = es.search(index=es_index, body=query)
    if_status_list = result['aggregations']['max_count']['buckets'];
    return if_status_list;


def check_1():
    name_count = "> <font color=#FF0000>**巡检接口访问量top10**</font>\n"
    num = 0
    data1 = ""
    count_list = get_count()
    for count_entity in count_list:
        num = num + 1
        top_name = "<font color=info>" + "top" + str(num) + "</font>"
        data1 = data1 + top_name  + ": " + count_entity['key'] + " " + "<font color=info>" + "调用次数: " + "</font>"  + str(count_entity['doc_count']) + "\n"
    return name_count + data1

def get_avg_timecost():
    es = Elasticsearch(hosts=es_hosts, port=9200, timeout=200)
    query =  {
      "aggs": {
        "avg_t": {
          "terms": {
            "field": "request.keyword",
            "order": {
              "avg_t2": "desc"
            },
            "size": 10
          },
          "aggs": {
            "avg_t2": {
              "avg": {
                "field": "request_time"
              }
            }
          }
        }
      },
      "size": 0,
      "_source": {
        "excludes": []
      },
      "stored_fields": [
        "*"
      ],
      "script_fields": {},
      "docvalue_fields": [
        {
          "field": "@timestamp",
          "format": "date_time"
        }
      ],
      "query": {
        "bool": {
          "must": [],
          "filter": [
            {
              "match_all": {}
            },
            {
              "exists": {
                "field": "query"
              }
            },
            {
              "bool": {
                "should": [
                  {
                    "match_phrase": {
                      "request": "/api"
                    }
                  }
                ],
                "minimum_should_match": 1
              }
            },
            {
              "range": {
                "@timestamp": {
                  "format": "strict_date_optional_time",
                  "gte": "now-24h",
                  "lte": "now"
                }
              }
            }
          ],
          "should": [],
          "must_not": [
            {
              "match_phrase": {
                "upstream_status": {
                  "query": "302"
                }
              }
            },
            {
              "bool": {
                "should": [
                  {
                    "match_phrase": {
                      "request": "/v1"
                    }
                  }
                ],
                "minimum_should_match": 1
              }
            }
          ]
        }
      }
    }
    result = es.search(index=es_index, body=query)
    avg_time_list = result['aggregations']['avg_t']['buckets'];
    return avg_time_list;


def check_2():
    name_count = "> <font color=#FF0000>**巡检接口平均耗时top10**</font>\n"
    num = 0
    data2 = ""
    count_list = get_avg_timecost()
    for count_entity in count_list:
        num = num + 1
        top_name = "<font color=info>" + "top" + str(num) + "</font>"
        avgt_numb = count_entity['doc_count']
        avg_time = round(count_entity['avg_t2']['value'],2)
        data2 = data2 + top_name  + ": " + count_entity['key'] + " " + "<font color=info>" + "总调用次数: " + "</font>" + str(avgt_numb)  + "<font color=info>" + " 平均耗时: " + "</font>"  + str(avg_time) + "\n"
    return name_count + data2

def get_fail():
    es = Elasticsearch(hosts=es_hosts, port=9200, timeout=200)
    query =  {
  "aggs": {
    "fail_t": {
      "terms": {
        "field": "request.keyword",
        "order": {
          "_count": "desc"
        },
        "size": 20
      },
      "aggs": {
        "fail_t1": {
          "terms": {
            "field": "status",
            "order": {
              "_count": "desc"
            },
            "size": 5
          }
        }
      }
    }
  },
  "size": 0,
  "_source": {
    "excludes": []
  },
  "stored_fields": [
    "*"
  ],
  "script_fields": {},
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    }
  ],
  "query": {
    "bool": {
      "must": [],
      "filter": [
        {
          "match_all": {}
        },
        {
          "exists": {
            "field": "query"
          }
        },
        {
          "bool": {
            "should": [
              {
                "match_phrase": {
                  "request": "/api"
                }
              }
            ],
            "minimum_should_match": 1
          }
        },
        {
          "range": {
            "@timestamp": {
              "format": "strict_date_optional_time",
              "gte": "now-24h",
              "lte": "now"
            }
          }
        }
      ],
      "should": [],
      "must_not": [
        {
          "match_phrase": {
            "status": {
              "query": "302"
            }
          }
        },
        {
          "bool": {
            "should": [
              {
                "match_phrase": {
                  "request": "grafana"
                }
              }
            ],
            "minimum_should_match": 1
          }
        }
      ]
    }
  }
}



    result = es.search(index=es_index, body=query)
    fail_list = result['aggregations']['fail_t']['buckets'];
    return fail_list;
def check_3():
    data3 = ''
    num = 0
    fail_dict = {}
    fail_numb = {}
    fail_list = get_fail()
    name_scu_odds = "<font color=#FF0000>**巡检接口失败率top10**</font>\n"
    for fail_entity in fail_list:
        fail_key = fail_entity['key']
        for status_list in fail_entity['fail_t1']['buckets']:
            if int(status_list['key']) == int(200):
                t_fail_odds = 1 - status_list['doc_count'] / fail_entity['doc_count']
                t_fail_numb = fail_entity['doc_count']
                fail_odds = round(t_fail_odds,2)
                fail_dict.update({fail_key:fail_odds})
                fail_numb.update({fail_key:t_fail_numb})
    fail_sort = sorted(fail_dict.items(),key = lambda x:x[1],reverse = True)[0:10]
    for sort_list in fail_sort:
        num = num + 1
        interface_name = sort_list[0]
        interface_odds = int(sort_list[1] * 100)
        interface_numb = int(fail_numb.get(interface_name))
        data3 = data3 + "<font color=info>" + "top"  + str(num) + ": " + "</font>" + sort_list[0] + " " + "<font color=info>" + "总调用次数: " + "</font>" + str(interface_numb) + "<font color=info>" + " 失败率: " + "</font>" + str(interface_odds) + "%"  + "\n"
    return name_scu_odds + data3

aa = check_1()
bb = check_2()
cc = check_3()
send_msg(str(aa))
send_msg(str(bb))
send_msg(str(cc))

效果图如下：