Prometheus
下载地址:https://prometheus.io/download/
安装 Prometheus
# 解压下载包
tar -zxvf prometheus-2.11.1.linux-amd64.tar.gz
# 做软链接
ln -s /usr/local/src/prometheus-2.11.1.linux-amd64 /usr/local/prometheus
配置 Prometheus
# 修改 prometheus.yml
###
alerting:
alertmanagers:
- static_configs:
- targets:
- 你的 alertmanager 服务器:9093
rule_files:
- "rules/*.yml"
*** 其余 prometheus 刮取策略就不贴出来了,这部分就是告警所需的模块
###
# 添加告警规则(在 prometheus.yml 同级目录下创建 rules 目录)
mkdir rules
vim alerts.yml
###
# 组告警
groups:
# 组名,报警规则组名称
- name: cert_alerts
rules:
- alert: certificateEW
expr: ceil((probe_ssl_earliest_cert_expiry - time())/86400) < 30
for: 1m
labels:
serverity: warning
annotations:
summary: The remaining time of the certificate is insufficient
console: You might want to Renewal certificate
###
# 配置 prometheus service
vim /etc/systemd/system/prometheus.service
###
ExecStart=/usr/local/prometheus/prometheus --
Documentation=https://prometheus.io/docs/introduction/overview/
[Unit]
Description=Prometheus Server
Documentation=https://prometheus.io/docs/introduction/overview/After=network.target
[Service]
Restart=on-failure
WorkingDirectory=/usr/local/prometheus/
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml
[Install]
WantedBy=multi-user.target
###
# 启动 prometheus
systemctl daemon-reload
systemctl start prometheus
Alertmanager
下载地址:https://prometheus.io/download/
安装
# 解压下载的包
tar -C /usr/local/alertmanager -zxvf alertmanager-0.21.0.linux-amd64.tar.gz
# 做软链接
ln -s /usr/local/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk /usr/local/bin/
cp /usr/local/alertmanager/alertmanager /usr/local/bin/
cp /usr/local/alertmanager/amtool /usr/local/bin/
配置
# 修改 alertmanager.yml
###
global:
# 每一分钟检查一次是否恢复
resolve_timeout: 1m
route:
# 设置默认接收人
receiver: 'webhook'
# 组告警等待时间,也就是告警产生后等待 10s
group_wait: 10s
#两组告警的间隔时间
group_interval: 10s
#重复告警的间隔时间,减少相同微信告警的发送频率
repeat_interval: 1h
#采用哪个标签来作为分组依据
group_by: ['alertname']
routes:
- receiver: webhook
group_wait: 10s
match:
team: node
receivers:
- name: 'webhook'
webhook_configs:
- url: http://localhost:8060/dingtalk/ops_dingding/send
send_resolved: true
###
# 配置 prometheus service
vim /etc/systemd/system/alertmanager.service
###
[unit]
Description=altermanager
Documentation=http://prometheus.io/docs/
[Service]
ExecStart=/usr/local/alertmanager/alertmanager --config.file=/usr/local/alertmanager/alertmanager.yml
Restart=on-failure
[Install]
WantedBy=multi-user.target
###
# 启动 alertmanager
systemctl daemon-reload
systemctl start prometheus
prometheus-webhook-dingtalk
下载地址:https://github.com/timonwong/prometheus-webhook-dingtalk/releases/tag/v1.4.0
安装
# 解压下载的包
tar -C /usr/local/prometheus-webhook-dingtalk -zxvf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz
# 做软链接
ln -s /usr/local/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk /usr/local/bin/
配置
# 配置 prometheus-webhook-dingtalk service
vim /etc/systemd/system/prometheus-webhook-dingtalk.service
###
[Unit]
Description=prometheus-webhook-dingtalk
After=network-online.target
[Service]
Restart=on-failure
ExecStart=/usr/local/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --ding.profile=ops_dingding=https://oapi.dingtalk.com/robot/send?access_token=XXXX(这里可以换成自己的webhook)
[Install]
WantedBy=multi-user.target
###
# 启动 prometheus-webhook-dingtalk 告警
systemctl daemon-reload
systemctl start prometheus-webhook-dingtalk
查看钉钉告警:已经正常告警