Centos7通过redis_exporter收集并发送到Prometheus以及使用Grafana进行展示并报警到钉钉&邮件

1. 前提：Centos7上安装redis-6.0.1.tar.gz

   启动：/opt/module/redis/bin/redis-server /opt/module/redis/bin/redis.conf
   查看：ps -ef | grep redis

   本人的启动命令如上

2. 下载prometheus-2.19.2.linux-amd64.tar.gz、grafana-7.0.6-1.x86_64.rpm、redis_exporter-v1.9.0.linux-amd64.tar.gz

3. [root@cdh632-worker02 soft]# cd /opt/soft
   [root@cdh632-worker02 soft]# tar -zxvf prometheus-2.19.2.linux-amd64.tar.gz -C /opt/module/
   [root@cdh632-worker02 soft]# tar -zxvf redis_exporter-v1.9.0.linux-amd64.tar.gz -C /opt/module/
   [root@cdh632-worker02 soft]# cd /opt/module
   由于无法联外网，故无法用yum安装rpm安装包时自定义安装目录，即：
   [root@cdh632-worker02 module]# yum -c /etc/yum.conf --installroot=/opt/module --releasever=/  install /opt/soft/grafana-7.0.6-1.x86_64.rpm
   故采用yum本地安装方式
   [root@cdh632-worker02 module]# yum localinstall grafana-7.0.6-1.x86_64.rpm
   [root@cdh632-worker02 module]# rpm -qa|grep grafana
   grafana-7.0.6-1.x86_64
   [root@cdh632-worker02 module]# rpm -ql grafana-7.0.6-1.x86_64
   /etc/grafana
   /etc/init.d/grafana-server
   /etc/sysconfig/grafana-server
   /usr/lib/systemd/system/grafana-server.service
   /usr/sbin/grafana-cli
   /usr/sbin/grafana-server
   /usr/share/grafana/VERSION
   ...
   [root@cdh632-worker02 module]# systemctl enable grafana-server
   Created symlink from /etc/systemd/system/multi-user.target.wants/grafana-server.service to /usr/lib/systemd/system/grafana-serve                                 r.service.
   [root@cdh632-worker02 module]# systemctl start grafana-server
   [root@cdh632-worker02 module]# groupadd prometheus
   [root@cdh632-worker02 module]# mv prometheus-2.19.2.linux-amd64 prometheus
   [root@cdh632-worker02 module]# useradd -g prometheus prometheus -d /opt/module/prometheus
   useradd: warning: the home directory already exists.
   Not copying any file from skel directory into it.
   [root@cdh632-worker02 module]# chown root:root prometheus
   [root@cdh632-worker02 module]# useradd -g prometheus prometheus -d /opt/module/prometheus
   useradd: user 'prometheus' already exists
   [root@cdh632-worker02 module]# cd prometheus/
   [root@cdh632-worker02 prometheus]# mkdir {data,cfg,logs,bin} -p
   [root@cdh632-worker02 prometheus]# mv prometheus promtool bin/
   [root@cdh632-worker02 prometheus]# mv prometheus.yml cfg/
   [root@cdh632-worker02 prometheus]# vi /etc/systemd/system/prometheus.service
   [Unit]
   Description=Prometheus
   Documentation=https://prometheus.io/
   After=network.target
   
   [Service]
   Type=simple
   User=prometheus
   ExecStart=/opt/module/prometheus/bin/prometheus --config.file=/opt/module/prometheus/cfg/prometheus.yml --storage.tsdb.path=/opt/module/prometheus/data
   Restart=on-failure
   
   [Install]
   WantedBy=multi-user.target
   [root@cdh632-worker02 prometheus]# vi /opt/module/prometheus/cfg/prometheus.yml
   # - targets: ['localhost:9090']
       # 注释掉上面一行，添加如下六行
       - targets: ['cdh632-worker02:9090']
     - job_name: 'redis'
   
       # metrics_path defaults to '/metrics'
       # scheme defaults to 'http'.
   
       static_configs:
       - targets: ['cdh632-worker02:9121']
   [root@cdh632-worker02 prometheus]# ll /etc/systemd/system/
   total 44
   drwxr-xr-x. 2 root root 4096 Mar  2 14:59 basic.target.wants
   lrwxrwxrwx. 1 root root   46 Mar  2 14:59 dbus-org.freedesktop.NetworkManager.service -> /usr/lib/systemd/system/NetworkManager.service
   lrwxrwxrwx. 1 root root   57 Mar  2 14:59 dbus-org.freedesktop.nm-dispatcher.service -> /usr/lib/systemd/system/NetworkManager-dispatcher.service
   lrwxrwxrwx. 1 root root   37 Mar  2 15:01 default.target -> /lib/systemd/system/multi-user.target
   drwxr-xr-x. 2 root root 4096 Mar  2 14:59 default.target.wants
   drwxr-xr-x. 2 root root 4096 Mar  2 14:59 getty.target.wants
   drwxr-xr-x. 2 root root 4096 Mar  2 14:59 local-fs.target.wants
   drwxr-xr-x. 2 root root 4096 Jul 11 17:01 multi-user.target.wants
   drwxr-xr-x. 2 root root 4096 Mar  2 14:59 network-online.target.wants
   -rw-r--r--. 1 root root  336 Jul 11 16:49 prometheus.service
   drwxr-xr-x. 2 root root 4096 Mar  2 17:35 sockets.target.wants
   drwxr-xr-x. 2 root root 4096 Mar  2 14:59 sysinit.target.wants
   drwxr-xr-x. 2 root root 4096 Mar  2 14:59 system-update.target.wants
   drwxr-xr-x. 2 root root 4096 Mar  2 14:59 vmtoolsd.service.requires
   [root@cdh632-worker02 prometheus]# cd ..
   [root@cdh632-worker02 module]# chown -R prometheus.prometheus prometheus
   [root@cdh632-worker02 prometheus]# systemctl daemon-reload
   [root@cdh632-worker02 prometheus]# systemctl enable prometheus
   Created symlink from /etc/systemd/system/multi-user.target.wants/prometheus.service to /etc/systemd/syst                                                         em/prometheus.service.
   [root@cdh632-worker02 prometheus]# systemctl start prometheus
   [root@cdh632-worker02 module]# systemctl status prometheus
   ● prometheus.service - Prometheus
      Loaded: loaded (/etc/systemd/system/prometheus.service; enabled; vendor preset: disabled)
      Active: active (running) since Sat 2020-07-11 17:09:36 CST; 2min 33s ago
        Docs: https://prometheus.io/
    Main PID: 3014 (prometheus)
      CGroup: /system.slice/prometheus.service
              └─3014 /opt/module/prometheus/bin/prometheus --config.file=/opt/module/prometheus/cfg/prometheus.yml --storage.tsdb.path=/...
   
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.680Z caller=main.go:678 msg="Starting TSDB ..."
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.680Z caller=web.go:524 component=web msg="S...0:9090
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.682Z caller=head.go:645 component=tsdb msg=...while"
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.682Z caller=head.go:706 component=tsdb msg=...ment=0
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.682Z caller=head.go:709 component=tsdb msg=…81.827µs
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.683Z caller=main.go:694 fs_type=EXT4_SUPER_MAGIC
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.683Z caller=main.go:695 msg="TSDB started"
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.683Z caller=main.go:799 msg="Loading config...us.yml
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.683Z caller=main.go:827 msg="Completed load...us.yml
   Jul 11 17:09:36 cdh632-worker02 prometheus[3014]: level=info ts=2020-07-11T09:09:36.683Z caller=main.go:646 msg="Server is read...ests."
   Hint: Some lines were ellipsized, use -l to show in full.
   

   打开其web ui：http://cdh632-worker02:9090/

[root@cdh632-worker02 module]# mv redis_exporter-v1.9.0.linux-amd64 redis_exporter
[root@cdh632-worker02 module]# vi /etc/systemd/system/redis_exporter.service
[Unit]
Description=redis_exporter
Documentation=https://prometheus.io/
After=network.target

[Service]
Type=simple
User=prometheus
ExecStart=/opt/module/redis_exporter/redis_exporter redis.addr=cdh632-worker02:6379
Restart=on-failure

[Install]
WantedBy=multi-user.target
[root@cdh632-worker02 module]# systemctl daemon-reload
[root@cdh632-worker02 module]# systemctl enable redis_exporter
[root@cdh632-worker02 module]# systemctl start redis_exporter
[root@cdh632-worker02 module]# systemctl status redis_exporter

Grafana的web ui：http://cdh632-worker02:3000/，默认账号/密码：admin/admin

复制：https://grafana.com/api/dashboards/763/revisions/1/download