CentOS7上docker部署prometheus+grafana

晴空06

已于 2022-10-24 20:27:42 修改

阅读量771

点赞数

分类专栏：测试工具性能测试文章标签： docker 运维 linux

于 2022-07-01 15:22:48 首次发布

本文链接：https://blog.csdn.net/sunnygirltest/article/details/125559057

版权

测试工具同时被 2 个专栏收录

28 篇文章 1 订阅

订阅专栏

性能测试

13 篇文章 3 订阅

订阅专栏

全部关闭防火墙和selinux：

systemctl stop firewalld && systemctl disable firewalld
sed -i 's/=enforcing/=disabled/g' /etc/selinux/config  && setenforce 0

安装docker

curl http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker.repo

yum makecache fast

yum install -y docker-ce

systemctl start docker && systemctl enable docker

node-export:

docker pull prom/node-exporter:latest

docker run -d -p 9100:9100 --name node-exporter prom/node-exporter:latest

docker ps

CONTAINER ID        IMAGE                       COMMAND                CREATED             STATUS              PORTS                    NAMES
95252704e558        prom/node-exporter:latest   "/bin/node_exporter"   3 seconds ago       Up 2 seconds        0.0.0.0:9100->9100/tcp   node-exporter

netstat -lntp |grep docker

tcp6       0      0 :::9100                 :::*                    LISTEN      9076/docker-proxy

alertmanager：

mkdir /home/prom && cd /home/prom

docker pull prom/alertmanager:latest

docker pull timonwong/prometheus-webhook-dingtalk               #钉钉告警

vim config.yml

targets:
  webhook:
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx             #修改为钉钉机器人的webhook
    mention:
      all: true

vim alertmanager.yml

global:
  resolve_timeout: 5m
  smtp_smarthost: 'smtp.163.com:465'             #邮箱smtp服务器代理，启用SSL发信, 端口一般是465
  smtp_from: 'alert@163.com'              #发送邮箱名称
  smtp_auth_username: 'alert@163.com'              #邮箱名称
  smtp_auth_password: 'password'                #邮箱密码或授权码
  smtp_require_tls: false

route:
  receiver: 'default'
  group_wait: 10s
  group_interval: 1m
  repeat_interval: 1h
  group_by: ['alertname']

inhibit_rules:
- source_match:
    severity: 'critical'
  target_match:
    severity: 'warning'
  equal: ['alertname', 'instance']
  
receivers:
- name: 'default'
  email_configs:
  - to: 'receiver@163.com'
    send_resolved: true
  webhook_configs:
  - url: 'http://192.168.20.218:8060/dingtalk/webhook/send'
    send_resolved: true

docker run -d -p 8060:8060 -v /home/prom/config.yml:/etc/prometheus-webhook-dingtalk/config.yml --name alertdingtalk timonwong/prometheus-webhook-dingtalk

docker run -d -p 9093:9093 -p 9094:9094 -v /home/prom/alertmanager.yml:/etc/alertmanager/alertmanager.yml --name alertmanager prom/alertmanager:latest

docker ps

CONTAINER ID        IMAGE                                   COMMAND                  CREATED             STATUS              PORTS                              NAMES
bed08b1b251d        prom/alertmanager:latest                "/bin/alertmanager -…"   3 seconds ago       Up 2 seconds        0.0.0.0:9093-9094->9093-9094/tcp   alertmanager
1449d7d0a445        timonwong/prometheus-webhook-dingtalk   "/bin/prometheus-web…"   21 seconds ago      Up 20 seconds       0.0.0.0:8060->8060/tcp             alertdingtalk
cf90ea931188        prom/node-exporter:latest               "/bin/node_exporter"     23 minutes ago      Up 23 minutes       0.0.0.0:9100->9100/tcp             node-exporter

netstat -lntp |grep docker

tcp6       0      0 :::9100                 :::*                    LISTEN      14970/docker-proxy  
tcp6       0      0 :::8060                 :::*                    LISTEN      17666/docker-proxy  
tcp6       0      0 :::9093                 :::*                    LISTEN      17808/docker-proxy  
tcp6       0      0 :::9094                 :::*                    LISTEN      17794/docker-proxy

prometheus：

mkdir -p /home/prom/data && chmod 777 /home/prom/data && cd /home/prom

docker pull prom/prometheus:latest

vim alert-rules.yml

groups:
  - name: node-alert
    rules:
    - alert: NodeDown
      expr: up{job="node"} == 0
      for: 5m
      labels:
        severity: critical
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} down"
        description: "Instance: {{ $labels.instance }} 已经宕机 5分钟"
        value: "{{ $value }}"
        
    - alert: NodeCpuHigh
      expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m]))) * 100 > 80
      for: 5m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} cpu使用率过高"
        description: "CPU 使用率超过 80%"
        value: "{{ $value }}"

    - alert: NodeCpuIowaitHigh
      expr: avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="iowait"}[5m])) * 100 > 50
      for: 5m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} cpu iowait 使用率过高"
        description: "CPU iowait 使用率超过 50%"
        value: "{{ $value }}"

    - alert: NodeLoad5High
      expr: node_load5 > (count by (instance) (node_cpu_seconds_total{job="node",mode='system'})) * 1.2
      for: 5m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} load(5m) 过高"
        description: "Load(5m) 过高，超出cpu核数 1.2倍"
        value: "{{ $value }}"

    - alert: NodeMemoryHigh
      expr: (1 - node_memory_MemAvailable_bytes{job="node"} / node_memory_MemTotal_bytes{job="node"}) * 100 > 90
      for: 5m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} memory 使用率过高"
        description: "Memory 使用率超过 90%"
        value: "{{ $value }}"

    - alert: NodeDiskRootHigh
      expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"}) * 100 > 90
      for: 10m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk(/ 分区) 使用率过高"
        description: "Disk(/ 分区) 使用率超过 90%"
        value: "{{ $value }}"

    - alert: NodeDiskBootHigh
      expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"}) * 100 > 80
      for: 10m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk(/boot 分区) 使用率过高"
        description: "Disk(/boot 分区) 使用率超过 80%"
        value: "{{ $value }}"

    - alert: NodeDiskReadHigh
      expr: irate(node_disk_read_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)
      for: 5m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk 读取字节数 速率过高"
        description: "Disk 读取字节数 速率超过 20 MB/s"
        value: "{{ $value }}"

    - alert: NodeDiskWriteHigh
      expr: irate(node_disk_written_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)
      for: 5m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk 写入字节数 速率过高"
        description: "Disk 写入字节数 速率超过 20 MB/s"
        value: "{{ $value }}"
        
    - alert: NodeDiskReadRateCountHigh
      expr: irate(node_disk_reads_completed_total{job="node"}[5m]) > 3000
      for: 5m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk iops 每秒读取速率过高"
        description: "Disk iops 每秒读取速率超过 3000 iops"
        value: "{{ $value }}"

    - alert: NodeDiskWriteRateCountHigh
      expr: irate(node_disk_writes_completed_total{job="node"}[5m]) > 3000
      for: 5m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk iops 每秒写入速率过高"
        description: "Disk iops 每秒写入速率超过 3000 iops"
        value: "{{ $value }}"

    - alert: NodeInodeRootUsedPercentHigh
      expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/"}) * 100 > 80
      for: 10m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk(/ 分区) inode 使用率过高"
        description: "Disk (/ 分区) inode 使用率超过 80%"
        value: "{{ $value }}"

    - alert: NodeInodeBootUsedPercentHigh
      expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/boot"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/boot"}) * 100 > 80
      for: 10m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} disk(/boot 分区) inode 使用率过高"
        description: "Disk (/boot 分区) inode 使用率超过 80%"
        value: "{{ $value }}"
        
    - alert: NodeFilefdAllocatedPercentHigh
      expr: node_filefd_allocated{job="node"} / node_filefd_maximum{job="node"} * 100 > 80
      for: 10m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} filefd 打开百分比过高"
        description: "Filefd 打开百分比 超过 80%"
        value: "{{ $value }}"

    - alert: NodeNetworkNetinBitRateHigh
      expr: avg by (instance) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8
      for: 3m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} network 接收比特数 速率过高"
        description: "Network 接收比特数 速率超过 20MB/s"
        value: "{{ $value }}"

    - alert: NodeNetworkNetoutBitRateHigh
      expr: avg by (instance) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8
      for: 3m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} network 发送比特数 速率过高"
        description: "Network 发送比特数 速率超过 20MB/s"
        value: "{{ $value }}"
        
    - alert: NodeNetworkNetinPacketErrorRateHigh
      expr: avg by (instance) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15
      for: 3m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} 接收错误包 速率过高"
        description: "Network 接收错误包 速率超过 15个/秒"
        value: "{{ $value }}"

    - alert: NodeNetworkNetoutPacketErrorRateHigh
      expr: avg by (instance) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15
      for: 3m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} 发送错误包 速率过高"
        description: "Network 发送错误包 速率超过 15个/秒"
        value: "{{ $value }}"

    - alert: NodeProcessBlockedHigh
      expr: node_procs_blocked{job="node"} > 10
      for: 10m
      labels:
        severity: warning
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} 当前被阻塞的任务的数量过多"
        description: "Process 当前被阻塞的任务的数量超过 10个"
        value: "{{ $value }}"

    - alert: NodeTimeOffsetHigh
      expr: abs(node_timex_offset_seconds{job="node"}) > 3 * 60
      for: 2m
      labels:
        severity: info
        instance: "{{ $labels.instance }}"
      annotations:
        summary: "instance: {{ $labels.instance }} 时间偏差过大"
        description: "Time 节点的时间偏差超过 3m"
        value: "{{ $value }}"

vim prometheus.yml

global:
  scrape_interval:     15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - 192.168.20.218:9093

rule_files:
  - "*rules.yml"
  
scrape_configs:
  - job_name: 'prometheus'
    static_configs:
    - targets: ['192.168.20.218:9090']

  - job_name: 'node'
    static_configs:
    - targets: ['192.168.20.211:9100','192.168.20.240:9100','192.168.20.218:9100']

  - job_name: 'alertmanager'
    static_configs:
    - targets: ['192.168.20.218:9093']

docker run -d -p 9090:9090 \
  -v /home/prom/prometheus.yml:/etc/prometheus/prometheus.yml \
  -v /home/prom/alert-rules.yml:/etc/prometheus/alert-rules.yml \
  -v /home/prom/data:/prometheus --name prometheus prom/prometheus:latest

docker ps

CONTAINER ID        IMAGE                       COMMAND                  CREATED             STATUS              PORTS                    NAMES
0a15519b970c        prom/prometheus:latest      "/bin/prometheus --c…"   3 seconds ago       Up 2 seconds        0.0.0.0:9090->9090/tcp   prometheus
95252704e558        prom/node-exporter:latest   "/bin/node_exporter"     16 hours ago        Up 16 hours         0.0.0.0:9100->9100/tcp   node-exporter

netstat -lntp |grep docker

tcp6       0      0 :::9090                 :::*                    LISTEN      9701/docker-proxy   
tcp6       0      0 :::9100                 :::*                    LISTEN      9076/docker-proxy

访问 http://192.168.20.218:9090
安装redis_exporter服务端(https://github.com/oliver006/redis_exporter)

docker run -d  --restart=always --name redis_exporter -p 9121:9121 oliver006/redis_exporter --redis.addr=192.168.20.240:6379 --redis.password=3vRgLpTMLHhMSCflL1iq

添加prometheus配置项

vim prometheus.yml

  - job_name: redis_exporter
    scrape_interval: 5s
    static_configs:
    - targets: ['192.168.20.218:9121']
      labels:
        instance: redis

重启prometheus

docker restart prometheus

安装influxdb(这里安装的是1.8.0版本的，2.0版本需要用户名密码设置)

docker run -d --name influxdb -p 8086:8086 -v /home/prom/influxdb_data:/var/lib/influxdb influxdb:1.8.10

安装完成后需要进入到容器里创建一个jmeter数据库

docker exec -it influxdb influx
Connected to http://localhost:8086 version 1.8.10
InfluxDB shell version: 1.8.10
> create database jmeter;
>exit

jmeter配置
在jmeter里添加“Backend Listener”，具体配置信息如下图:

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-kz8LOPBI-1656659097931)(jmeter-influxdb.jpg)]

在grafana里导入5496模板

安装grafana:

mkdir -p /home/prom/grafana && chmod 777 /home/prom/grafana

docker pull grafana/grafana:latest

docker run -d -p 3000:3000 -v /home/prom/grafana:/var/lib/grafana --name=grafana grafana/grafana:latest

docker ps

CONTAINER ID        IMAGE                       COMMAND                CREATED             STATUS              PORTS                    NAMES
17d5c5d415b5        grafana/grafana:latest      "/run.sh"              4 seconds ago       Up 3 seconds        0.0.0.0:3000->3000/tcp   grafana
bd53df9eb9fa        prom/node-exporter:latest   "/bin/node_exporter"   16 hours ago        Up 16 hours         0.0.0.0:9100->9100/tcp   node-exporter

netstat -lntp |grep docker

tcp6       0      0 :::9100                 :::*                    LISTEN      6558/docker-proxy   
tcp6       0      0 :::3000                 :::*                    LISTEN      13123/docker-proxy

grafana容器配置文件：/etc/grafana/grafana.ini。

访问IP:3000，初始账号密码为admin/admin，会要求更改密码。

选择数据源Prometheus——http://ip:9090，导入模板8919，

Grafana模板ID	描述
8919	node_export
11835	redis_export
5496	jmeter_dashboards
10467、14694、14510	windows_exprot

报错：dial tcp 192.168.20.218:9093: connect: no route to host

解决方法：

iptables -P INPUT ACCEPT
iptables -P FORWARD ACCEPT
iptables -P OUTPUT ACCEPT
iptables -F

如果上面操作完还是不行那就是路由信息配置有问题了，使用命令查看路由信息（主要是网关和DNS配置信息）

ip route

确认网关与DNS配置没有问题，如果有问题可以修改“网上信息”。如ens192网卡

vim /etc/sysconfig/network-scripts/ifcfg-ens192

监测每台服务都需要安装node_export

docker run -d -p 9100:9100 --name node-exporter prom/node-exporter:latest

3.监测每台服务都需要安装redis_export

nohup ./redis_exporter -redis.addr=192.168.20.240:6379  -redis.password=3vRgLpTMLHhMSCflL1iq -web.listen-address :9121 &

cat > /usr/lib/systemd/system/redis_exporter.service  <<EOF
[Unit]
Description=redis_exporter
Documentation=https://github.com/oliver006/redis_exporter
After=network.target
[Service]
Type=simple
User=prometheus
ExecStart=/usr/local/redis_exporter/redis_exporter -redis.addr 192.168.20.240:6379  -redis.password 3vRgLpTMLHhMSCflL1iq
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF