kibana+sentinl 6.2.4实现钉钉邮件告警

一.ELK安装
1.软件架构：filebeat----elasticsearch----kibana+sentinel---(邮件和钉钉)
2.软件下载地址：https://www.elastic.co/cn/downloads/past-releases# （本教程使用的为6.2.4）
3.elasticsearch安装
[root@VM_0_7_centos ~]# egrep -v "^$|^#" /opt/app/elasticsearch-6.2.4/config/elasticsearch.yml

cluster.name: globalglb-elk
node.name: globalglb
network.host: 0.0.0.0
http.port: 9200
http.cors.enabled: true
http.cors.allow-origin: "*"

4.kibana安装
[root@VM_0_10_centos ~]# egrep -v "^$|^#" /opt/app/kibana-6.2.4-linux-x86_64/config/kibana.yml

server.port: 5601
server.host: "10.9.0.10"
elasticsearch.url: "http://10.9.0.7:9200"
sentinl:
  settings:
    email:
      active: true
      user: elkmonitor@126.com
      password: YOUxin2019
      host: smtp.126.com
      ssl: false
    report:
      active: true

4.filebeat配置信息
#cat ffilebeat.yml

filebeat.prospectors:
########################
- input_type: log
  paths:
    - /opt/app/logs/evolut-api-gateway/evolut-api-gateway.log
  #json.keys_under_root: true
  #json.overwrite_keys: true
  fields:
    index: 'prd-evolut-api-gateway'
  exclude_lines: ['^$']
  multiline:
    pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
    negate: true
    match: after
#############################
- input_type: log
  paths:
    - /opt/app/logs/evolut-file-service/evolut-file-service.log
  #json.keys_under_root: true
  #json.overwrite_keys: true
  fields:
    index: 'prd-evolut-file-service'
  exclude_lines: ['^$']
  multiline:
    pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
    negate: true
    match: after
#####################################
- input_type: log
  paths:
    - /opt/app/logs/evolut-admin/evolut-admin.log 
  #json.keys_under_root: true
  #json.overwrite_keys: true
  fields:
    index: 'prd-evolut-admin'
  exclude_lines: ['^$']
  multiline:
    pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
    negate: true
    match: after
##################################
- input_type: log
  paths:
    - /opt/app/logs/evolut-insurance/evolut-insurance.log 
  #json.keys_under_root: true
  #json.overwrite_keys: true
  fields:
    index: 'prd-evolut-insurance'
  exclude_lines: ['^$']
  multiline:
    pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
    negate: true
    match: after
#########################
- input_type: log
  paths:
    - /opt/app/logs/evolut-message/evolut-message.log 
  #json.keys_under_root: true
  #json.overwrite_keys: true
  fields:
    index: 'prd-evolut-message'
  exclude_lines: ['^$']
  multiline:
    pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
    negate: true
    match: after
####################
- input_type: log
  paths:
    - /opt/app/logs/evolut-schedule/evolut-schedule.log
  #json.keys_under_root: true
  #json.overwrite_keys: true
  fields:
    index: 'prd-evolut-schedule'
  exclude_lines: ['^$']
  multiline:
    pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
    negate: true
    match: after
############
- input_type: log
  paths:
    - /opt/app/logs/evolut-user/evolut-user.log
  #json.keys_under_root: true
  #json.overwrite_keys: true
  fields:
    index: 'prd-evolut-user'
  exclude_lines: ['^$']
  multiline:
    pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
    negate: true
    match: after
####################
####################
- input_type: log
  paths:
    - /opt/app/logs/evolut-esign/evolut-esign.log
  #json.keys_under_root: true
  #json.overwrite_keys: true
  fields:
    index: 'prd-evolut-esign'
  exclude_lines: ['^$']
  multiline:
    pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
    negate: true
    match: after
###################
output.elasticsearch:
  hosts: ["10.9.0.7:9200"]
  indices:
    - index: "prd-evolut-file-service-%{+YYYY.MM.dd}"
      when.contains:
        fields:
          index: "prd-evolut-file-service"
    - index: "prd-evolut-api-gateway-%{+YYYY.MM.dd}"
      when.contains:
        fields:
          index: "prd-evolut-api-gateway"
    - index: "prd-evolut-admin-%{+YYYY.MM.dd}"
      when.contains:
        fields:
          index: "prd-evolut-admin"
    - index: "prd-evolut-insurance-%{+YYYY.MM.dd}"
      when.contains:
        fields:
          index: "prd-evolut-insurance"
    - index: "prd-evolut-message-%{+YYYY.MM.dd}"
      when.contains:
        fields:
          index: "prd-evolut-message"
    - index: "prd-evolut-schedule-%{+YYYY.MM.dd}"
      when.contains:
        fields:
          index: "prd-evolut-schedule"
    - index: "prd-evolut-user-%{+YYYY.MM.dd}"
      when.contains:
        fields:
          index: "prd-evolut-user"
    - index: "prd-evolut-esign-%{+YYYY.MM.dd}"
      when.contains:
        fields:
          index: "prd-evolut-esign"

备注：filebeat监控多个文件，根据不同的文件家里索引

二、配置kibana+sentnl邮件和钉钉告警
1.登录控制台直接导入下面的代码，根据修改改

{
  "actions": {
    "邮件告警": {
      "name": "日志异常",
      "throttle_period": "0h2m0s",
      "email_html": {
        "stateless": false,
        "subject": "evolut-api-gateway模块--ERROR日志",
        "priority": "medium",
        "html": "<p><i>Hi,各位同事请注意下面有 {{payload.hits.total}} 条错误信息，请查看并处理！！</i>.</p>\n<div style=\"color:grey;\">\n  <hr />\n</div>\n<div>\n<br>{{#payload.hits.hits}} <li style='color:red'><b>source:</b> {{_source.source}} </li><br><li><b>message</b>: {{_source.message}}</li><br><br>{{/payload.hits.hits}}  \n</div>",
        "to": "xiong@xxx.com",
        "from": "e@126.com"
      }
    },
    "钉钉告警模板": {
      "name": "webhook告警",
      "throttle_period": "0h2m0s",
      "webhook": {
        "priority": "medium",
        "stateless": false,
        "method": "POST",
        "host": "oapi.dingtalk.com",
        "port": "443",
        "path": "/robot/send?access_token=bdf86156bcded8b10727ceff898b943ef726baaebd797f760336",
        "body": "{\r\n    \"msgtype\": \"markdown\",\r\n    \"at\": {\r\n        \"isAtAll\": \"True\"\r\n    },\r\n    \"markdown\": {\r\n        \"title\": \"异常消息\",\r\n        \"text\": \" evolut-api-gateway模块-错误日志: \\n {{#payload.hits.hits}} {{_source.message}} \r\n{{/payload.hits.hits}}\"\r\n    }\r\n}",
        "params": {
          "watcher": "{{watcher.title}}",
          "payload_count": "{{payload.hits.total}}"
        },
        "headers": {
          "Content-Type": "application/json"
        },
        "message": "生产环境异常",
        "use_https": true
      }
    }
  },
  "input": {
    "search": {
      "request": {
        "index": [
          "prd-evolut-api-gateway*"
        ],
        "body": {
          "query": {
            "bool": {
              "must": {
                "match": {
                  "message": "ERROR"
                }
              },
              "filter": {
                "range": {
                  "@timestamp": {
                    "gte": "now-5m/m",
                    "lte": "now/m",
                    "format": "epoch_millis"
                  }
                }
              }
            }
          },
          "size": 2,
          "aggs": {
            "dateAgg": {
              "date_histogram": {
                "field": "@timestamp",
                "time_zone": "Asia/Shanghai",
                "interval": "1m",
                "min_doc_count": 1
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "script": "payload.hits.total >= 1"
    }
  },
  "transform": {},
  "trigger": {
    "schedule": {
      "later": "every 2 minutes"
    }
  },
  "disable": false,
  "report": false,
  "title": "evolut-api-gateway"
}

邮件告警内容

告警邮件

钉钉告警
登录钉钉-新建群组--选择机器人

企业微信告警：
1.登录企业微信-创建群聊-创建机器人

2.在kibana的Sentinl里面的错误日志告警添加webhook

备注：
Boby代码如下

{
    "msgtype": "markdown",
    "markdown": {
        "content": "各位同事请注意下面有{{payload.hits.total}}条错误信息。\n
         >微服务模块:evolut-insurance\n
         >日志类型:ERROR \n
         >错误日志内容:{{#payload.hits.hits}}{{_source.message}}{{/payload.hits.hits}}"
    }
}

HEaders代码如下

{
  "Content-Type": "application/json"
}

错误日志如下