Prometheus+node_exporter+Alertmanager+钉钉机器人webhook，实现当机器宕机发送钉钉消息通知用户

1、node_exporter（crentos安装）
（1）下载node_exporter-0.18.1.linux-amd64.tar.gz后
解压：tar -zxvf node_exporter-0.18.1.linux-amd64.tar.gz
（2）在node_exporter-0.18.1.linux-amd64目录下
启动命令:nohup ./node_exporter &

2、Alertmanager（crentos安装）
（1）下载Alertmanager后，把alertmanager.yml里面的内容全删除，然后添加如下内容：

#同样的警告5分钟后不会收到
global:
  resolve_timeout: 5m
#路由规则
route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 1m
  repeat_interval: 4h
  receiver: 'webhook'
#接收者这里的ip是你启动springboot的ip地址
receivers:
- name: 'webhook'
  webhook_configs:
- url: 'http://ip:8080/demo'

（2）在安装目录下，启动命令：./alertmanager

3、Prometheus（crentos安装）
（1）下载prometheus-2.28.1.linux-amd64.tar.gz后
把prometheus.yml的内容替换成下面内容：

# my global config
global:
  scrape_interval:     15s # 采样周期
  evaluation_interval: 15s # 告警规则计算周期

# Alertmanager 的配置
alerting:
  alertmanagers:
  - static_configs:
   # 这个ip是Alertmanager的地址
    - targets: ['ip:9093']

# 报警规则文件可以指定多个，并且可以使用通配符*
rule_files:
  - "rules/*_rules.yml"

# 采集job配置
scrape_configs:
  - job_name: 'prometheus'
    #使用file_sd_configs热加载配置文件
    file_sd_configs:
      #指定1分钟加载一次配置
    - refresh_interval: 1m
      files:
      - config_prometheus.json
  - job_name: 'node_exporter'
    metrics_path: "/metrics"
    scrape_interval: 5s
    static_configs:
    #这个ip是node_exporter的ip地址
    - targets: ['ip:9100']

（2）添加config_prometheus.json

[
  {
    "targets": [ "localhost:9090"],
    "labels": {
      "service": "promethenus"
    }
  }
]

（3）编写报警规则rules/host_rules.yml

groups:
- name: example
  rules:
  # 对于任何无法访问> 1分钟的实例的警报。
  - alert: InstanceDown
    expr: up == 0
    for: 1m
    labels:
      severity: page
    annotations:
      summary: "Instance {{ $labels.instance }} down"
      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."

（4）安装目录下 启动命令：./prometheus
4、钉钉的webhook整合springboot
（1）实现代码

package com.hjl.test.controller;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.tomcat.util.codec.binary.Base64;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import com.dingtalk.api.DefaultDingTalkClient;
import com.dingtalk.api.DingTalkClient;
import com.dingtalk.api.request.OapiRobotSendRequest;
import com.dingtalk.api.response.OapiRobotSendResponse;
import com.taobao.api.ApiException;
import org.apache.tomcat.util.codec.binary.Base64;

import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.net.URLEncoder;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.Map;



@Controller
@RequestMapping("/")
public class AlertController {


    @RequestMapping(value = "/demo",produces = "application/json;charset=UTF-8")
    @ResponseBody
    public String pstn(@RequestBody String json) {
        Map<String, Object> result = new HashMap<>();
        result.put("msg", "报警成功");
        result.put("code", 0);
        if(StringUtils.isBlank(json)){
            return JSON.toJSONString(result);
        }
        JSONObject jo = JSON.parseObject(json);
        //获取json中的commonAnnotations
        JSONObject commonAnnotations = jo.getJSONObject("commonAnnotations");
        String status = jo.getString("status");
        if (commonAnnotations == null) {
            return JSON.toJSONString(result);
        }
        String subject = commonAnnotations.getString("summary");
        String content = commonAnnotations.getString("description");
        //钉钉的安装规则是通过签名
        Long timestamp = System.currentTimeMillis();
        //这个secret是钉钉机器人里面选择签名安全规则后，会给你的
        String secret="SEC****";
        String sign=getsign(secret,timestamp);
        //下面的access_token写上钉钉机器人里面那个
        String url="https://oapi.dingtalk.com/robot/send?access_token=***&timestamp="+timestamp+"&sign="+sign;
        DingTalkClient client = new DefaultDingTalkClient(url);
        OapiRobotSendRequest request = new OapiRobotSendRequest();
        request.setMsgtype("text");
//OapiRobotSendRequest.Text对象用来添加文本
        OapiRobotSendRequest.Text text=new OapiRobotSendRequest.Text();
//OapiRobotSendRequest.Text对象用来@我们要通知的用户
OapiRobotSendRequest.At at = new OapiRobotSendRequest.At();
        text.setContent(content);
        request.setText(text);
        at.setIsAtAll(false);
        request.setAt(at);
        try {
            OapiRobotSendResponse response = client.execute(request);
        } catch (ApiException e) {
            e.printStackTrace();
        }
        return JSON.toJSONString(result);
    }
//获取签名的方法
    public static  String getsign(String secret,Long stamp){
        String stringToSign = stamp + "\n" + secret;
        Mac mac = null;
        String sign=null;
        try {
            mac = Mac.getInstance("HmacSHA256");
            mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256"));
            byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8"));
            sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)),"UTF-8");
        } catch (Exception e) {
            e.printStackTrace();
        }
        return sign;
    }
}

（2）pom文件中的配置

  <dependencies>
        <!--lombok-->
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.12.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.70</version>
        </dependency>

        <!-- springboot的web启动依赖-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <!--这个是钉钉机器人服务器后台项目lib包下的包，把他的lib整个负责到你的项目下就可以了-->
        <dependency>
            <groupId>com.dingtalk</groupId>
            <artifactId>dingtalk-api-sdk</artifactId>
            <version>1.0.0-SNAPSHOT</version>
            <scope>system</scope>
            <systemPath>${pom.basedir}/lib/taobao-sdk-java-auto_1479188381469-20210207.jar</systemPath>
        </dependency>
    </dependencies>

5、测试：先启动node_exporter,然后启动Alertmanager，再去启动Prometheus，最后启动你的springboot项目，都启动成功后，关闭node_exporter的机器，Prometheus检测到机器宕机就会发送警告。