Prometheus 配置钉钉报警

配置钉钉机器人

获取token
https://oapi.dingtalk.com/robot/send?access_token=a85f438fa00c451c0b32ed0fc799609093436f07e93e518bde59dd6d26f930e9

配置钉钉告警插件

vim dingtalk-webhook.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: prometheus-webhook-dingtalk
  name: prometheus-webhook-dingtalk
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus-webhook-dingtalk
  template:
    metadata:
      labels:
        app: prometheus-webhook-dingtalk
    spec:
      containers:
      - name: prometheus-webhook-dingtalk
        image: timonwong/prometheus-webhook-dingtalk:v0.3.0
        imagePullPolicy: IfNotPresent
        args:
          - --ding.profile=webhook1=获取的钉钉token
          - --template.file=/usr/share/prometheus-webhook-dingtalk/template/webhook-dingtalk.tmpl
        volumeMounts:
        - mountPath: /usr/share/prometheus-webhook-dingtalk/template/
          name: webhook-dingtalk-template
        ports:
        - containerPort: 8060
          protocol: TCP
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            cpu: 200m
            memory: 1000Mi
      volumes:
      - name: webhook-dingtalk-template
        configMap:
          name: webhook-dingtalk-template
          defaultMode: 420

---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: prometheus-webhook-dingtalk
  name: prometheus-webhook-dingtalk
  namespace: monitoring
spec:
  ports:
  - port: 8060
    protocol: TCP
    targetPort: 8060
  selector:
    app: prometheus-webhook-dingtalk
  sessionAffinity: None

配置报警模板

mkdir -p /usr/share/prometheus-webhook-dingtalk/template/
vim /usr/share/prometheus-webhook-dingtalk/template/webhook-dingtalk.tmpl 
{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
{{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}

{{ define "__text_alert_list" }}{{ range . }}

**报警内容**
{{ range .Labels.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}
{{ end }}

**报警信息**
{{ range .Annotations.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}
{{ end }}


{{ end }}{{ end }}

{{ define "ding.link.title" }}{{ template "__subject" . }}{{ end }}
{{ define "ding.link.content" }}

### k8s集群测试环境监控报警

---

{{ index .GroupLabels "alertname" }}
{{ template "__text_alert_list" .Alerts.Firing }}
{{ end }}


{{ if gt (len .Alerts.Resolved) 0 -}}

**恢复内容**
{{ template "__text_resolve_list" .Alerts.Resolved }}
{{ end }}

创建configmap 报警模板生成

kubectl create configmap webhook-dingtalk-template --from-file=webhook-dingtalk.tmpl -n monitoring

应用dingtalk-webhook.yaml

kubectl apply -f dingtalk-webhook.yaml

添加告警接受器

vim kube-prometheus-0.9.0/manifests/alertmanager.yaml
global:
  resolve_timeout: 5m
route:
  group_by: ['job']
  group_wait: 0s
  group_interval: 5m
  repeat_interval: 2h
  receiver: webhook
receivers:
- name: 'webhook'
  webhook_configs:
  - url: 'http://prometheus-webhook-dingtalk:8060/dingtalk/webhook1/send'
    send_resolved: true

注：上述配置url: 'http://webhook-dingtalk.monitoring.svc.cluster.local:8060/dingtalk/webhook1/send' 是dingtalk-webhook.yaml文件中svc的地址。

替换原有secret

cd kube-prometheus-0.9.0/manifests/
kubectl delete  secret alertmanager-main -n monitoring
kubectl create  secret generic alertmanager-main --from-file=alertmanager.yaml -n monitoring

测试

#! /bin/bash

alert_payload='[
  {
    "labels": {
       "alertname": "报警通道测试",
       "dev": "sda1",
       "instance": "127.0.0.1",
       "severity": "critical"
     },
     "annotations": {
        "info": "报警通道测试",
        "summary": "报警通道测试"
      }
  }
]'

alertmanager_0_ip=`kubectl describe po alertmanager-main-0 -n monitoring | grep "IP:     " |  awk 'END{print $2}'`

curl -XPOST -H "Content-Type: application/json" -d"${alert_payload}" http://${alertmanager_0_ip}:9093/api/v1/alerts