Prometheus Alertmanager集成钉钉告警

Alertmanager下载安装

• Alertmanager下载

//下载
wget https://github.com/prometheus/alertmanager/releases/download/v0.26.0/alertmanager-0.26.0.linux-amd64.tar.gz

//解压
tar -zxvf alertmanager-0.26.0.linux-amd64.tar.gz

• 配置策略

vi alertmanager.yml

route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 1h
  repeat_interval: 1h
  receiver: 'webhook1'
  routes:
  - receiver: 'webhook1'
    match_re:
      alertname: ".*" #所有告警都走webhook1
receivers:
  - name: 'webhook1'
    webhook_configs:
    - url: 'http://193.168.9.103:8060/dingtalk/webhook1/send' #钉钉Webhook-dingtalk ip和端口
      send_resolved: true
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

• 启动alertmanager

nohup ./alertmanager --config.file="alertmanager.yml" 2>&1 &

//确认9093端口是否启动，此时命令应该有输出。
ss -alntup | grep -i 9093      

• 登录： http://localhost:9093/

• Prometheus配置alertmanager

alerting:
  alertmanagers:
  - static_configs:
    - targets: ['193.168.9.103:9093']

Webhook-dingtalk安装启动

• Webhook-dingtalk下载

//下载
wget https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v2.1.0/prometheus-webhook-dingtalk-2.1.0.linux-amd64.tar.gz
//解压
tar -zxvf prometheus-webhook-dingtalk-2.1.0.linux-amd64.tar.gz 

• 添加钉钉告警模板

vi template.tmpl

{{ define "__subject" }}
[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]
{{ end }}
 
 
{{ define "__alert_list" }}{{ range . }}
---
{{ if .Labels.owner }}@{{ .Labels.owner }}{{ end }}
 
**告警主题**: {{ .Annotations.summary }}

**告警类型**: {{ .Labels.alertname }}
 
**告警级别**: {{ .Labels.severity }} 
 
**告警主机**: {{ .Labels.instance }} 
 
**告警信息**: {{ index .Annotations "description" }}
 
**告警时间**: {{ dateInZone "2006.01.02 15:04:05" (.StartsAt) "Asia/Shanghai" }}
{{ end }}{{ end }}
 
{{ define "__resolved_list" }}{{ range . }}
---
{{ if .Labels.owner }}@{{ .Labels.owner }}{{ end }}

**告警主题**: {{ .Annotations.summary }}

**告警类型**: {{ .Labels.alertname }} 
 
**告警级别**: {{ .Labels.severity }}
 
**告警主机**: {{ .Labels.instance }}
 
**告警信息**: {{ index .Annotations "description" }}
 
**告警时间**: {{ dateInZone "2006.01.02 15:04:05" (.StartsAt) "Asia/Shanghai" }}
 
**恢复时间**: {{ dateInZone "2006.01.02 15:04:05" (.EndsAt) "Asia/Shanghai" }}
{{ end }}{{ end }}
 
 
{{ define "default.title" }}
{{ template "__subject" . }}
{{ end }}
 
{{ define "default.content" }}
{{ if gt (len .Alerts.Firing) 0 }}
**====侦测到{{ .Alerts.Firing | len  }}个故障====**
{{ template "__alert_list" .Alerts.Firing }}
---
{{ end }}
 
{{ if gt (len .Alerts.Resolved) 0 }}
**====恢复{{ .Alerts.Resolved | len  }}个故障====**
{{ template "__resolved_list" .Alerts.Resolved }}
{{ end }}
{{ end }}
 
 
{{ define "ding.link.title" }}{{ template "default.title" . }}{{ end }}
{{ define "ding.link.content" }}{{ template "default.content" . }}{{ end }}
{{ template "default.title" . }}
{{ template "default.content" . }}

• 配置钉钉webhook地址

vi webhook-dingtalk/config.yml


templates:
  - ./template.tmpl ##模板地址

## You can also override default template using `default_message`
## The following example to use the 'legacy' template from v0.3.0
#default_message:
#  title: '{{ template "legacy.title" . }}'
#  text: '{{ template "legacy.content" . }}'

## Targets, previously was known as "profiles"
targets:
  webhook1:
    url: https://oapi.dingtalk.com/robot/send?access_token=6aca15b2e98fc627bdee45c0331beea84c2fa94abec8345aba0cd54f8d2cfad3
  webhook_legacy:
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    # Customize template content
    message:
      # Use legacy template
      title: '{{ template "legacy.title" . }}'
      text: '{{ template "legacy.content" . }}'
  webhook_mention_all:
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    mention:
      all: true
  webhook_mention_users:
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    mention:
      mobiles: ['156xxxx8827', '189xxxx8325']

• 配置Alertmanager使用systemd管理

vim /etc/systemd/system/prometheus-webhook-dingtalk.service

[Unit]
Description=prometheus-webhook-dingtalk
After=network-online.target

[Service]
Restart=on-failure
ExecStart=/opt/webhook-dingtalk/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --web.listen-address=:8060  --config.file=/opt/webhook-dingtalk/prometheus-webhook-dingtalk/config.yml

[Install]
WantedBy=multi-user.target

• 启动

systemctl start prometheus-webhook-dingtalk.service

netstat -anput |grep 8060

//开机自启
systemctl enable prometheus-webhook-dingtalk.service --now