prometheus+grafana+alertmanager邮件

docker pull prom/node-exporter
docker pull prom/prometheus
docker pull grafana/grafana

#C端监控
docker run -d -p 9100:9100 \
  --restart=always \
  -e TZ=Asia/Shanghai \
  -v "/proc:/host/proc:ro" \
  -v "/sys:/host/sys:ro" \
  -v "/:/rootfs:ro" \
  prom/node-exporter

#prometheus
docker run -d --name prometheus --restart=always -p 9090:9090  -e TZ=Asia/Shanghai -v /home/prometheus/:/etc/prometheus/ prom/prometheus

#grafana
docker run -d \
  -p 3000:3000 \
  --name=grafana \
  -v /home/monitor/grafana:/var/lib/grafana \
  grafana/grafana

#监控邮件
docker run -d --name alertmanager --restart=always -e TZ=Asia/Shanghai -p 9093:9093 -v /home/monitor/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml prom/alertmanager:latest

#规则rules
groups:
- name: 服务器资源监控
  rules:
  - alert: 内存使用率过高
    expr: (node_memory_Buffers_bytes+node_memory_Cached_bytes+node_memory_MemFree_bytes)/node_memory_MemTotal_bytes*100 > 5
    for: 15s  # 告警持续时间，超过这个时间才会发送给alertmanager
    labels:
      severity: 严重告警
    annotations:
      summary: "{{ $labels.instance }} 内存使用率过高，请尽快处理！"
      description: "{{ $labels.instance }}内存使用率超过90%,当前使用率{{ $value }}%."
          
  - alert: 服务器宕机
    expr: up == 0
    for: 3m
    labels:
      severity: 严重告警
    annotations:
      summary: "{{$labels.instance}} 服务器宕机，请尽快处理！"
      description: "{{$labels.instance}} 服务器延时超过3分钟，当前状态{{ $value }}. "
 
  - alert: CPU高负荷
    expr: 100 - (avg by (instance,job)(irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
    for: 5m
    labels:
      severity: 严重告警
    annotations:
      summary: "{{$labels.instance}} CPU使用率过高，请尽快处理！"
      description: "{{$labels.instance}} CPU使用大于90%，当前使用率{{ $value }}%. "
      
  - alert: 磁盘IO性能
    expr: avg(irate(node_disk_io_time_seconds_total[1m])) by(instance,job)* 100 > 90
    for: 5m
    labels:
      severity: 严重告警
    annotations:
      summary: "{{$labels.instance}} 流入磁盘IO使用率过高，请尽快处理！"
      description: "{{$labels.instance}} 流入磁盘IO大于90%,当前使用率{{ $value }}%."
 
 
  - alert: 网络流入
    expr: ((sum(rate (node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance,job)) / 100) > 102400
    for: 5m
    labels:
      severity: 严重告警
    annotations:
      summary: "{{$labels.instance}} 流入网络带宽过高，请尽快处理！"
      description: "{{$labels.instance}} 流入网络带宽持续5分钟高于100M. RX带宽使用量{{$value}}."
 
  - alert: 网络流出
    expr: ((sum(rate (node_network_transmit_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance,job)) / 100) > 102400
    for: 5m
    labels:
      severity: 严重告警
    annotations:
      summary: "{{$labels.instance}} 流出网络带宽过高,请尽快处理！"
      description: "{{$labels.instance}} 流出网络带宽持续5分钟高于100M. RX带宽使用量{$value}}."
  
  - alert: TCP连接数
    expr: node_netstat_Tcp_CurrEstab > 10000
    for: 2m
    labels:
      severity: 严重告警
    annotations:
      summary: " TCP_ESTABLISHED过高！"
      description: "{{$labels.instance}} TCP_ESTABLISHED大于100%,当前使用率{{ $value }}%."
 
  - alert: 磁盘容量
    expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 90
    for: 1m
    labels:
      severity: 严重告警
    annotations:
      summary: "{{$labels.mountpoint}} 磁盘分区使用率过高，请尽快处理！"
      description: "{{$labels.instance}} 磁盘分区使用大于90%，当前使用率{{ $value }}%."

prometheus重新挂载
docker run -d --name prometheus --restart=always -p 9090:9090  -e TZ=Asia/Shanghai -v /home/monitor/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml  -v /home/monitor/alertmanager/rules.yml/:/usr/local/alertmanager/rules/*.yml  prom/prometheus

alertmanager重新挂载   模板有限不成功
docker run -d --name alertmanager --restart=always -e TZ=Asia/Shanghai -p 9093:9093 -v /home/monitor/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml -v /home/monitor/prometheus/alertmanager-tmpl/email.tmpl:/etc/alertmanager/email.tmpl prom/alertmanager:latest
  

#C端容器监控
docker run --name cadvisor -d -p 8090:8080 \
--restart=always \
-e TZ=Asia/Shanghai
-v /:/rootfs:ro \
-v /var/run:/var/run:rw \
-v /sys:/sys:ro \
-v /var/lib/docker/:/var/lib/docker:ro \
google/cadvisor