构建企业级监控大屏 prometheus + grafana

1 prometheus下载安装

1.1 虚机部署

https://prometheus.io/download/

wget https://github.com/prometheus/prometheus/releases/download/v2.53.0/prometheus-2.53.0.linux-amd64.tar.gz
tar -xvf prometheus-2.53.0.linux-amd64.tar.gz
cd /home/prometheus-2.53.0.linux-amd64
./prometheus 

1.2 容器部署

# 1下载
docker pull prom/prometheus
# 2创建prometheus配置文件
mkdir /opt/prometheus
cd /opt/prometheus/
vim prometheus.yml
# 3启动
docker run -d --name prometheus --restart=always -p 9090:9090 -v /opt/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus

1.3 prometheus.yml配置

scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"
    static_configs:
      - targets: ["9.135.95.39:9090"]

访问地址：http://127.0.0.1:9090/targets?search=

2 grafana下载安装

2.1 虚机部署

Download Grafana | Grafana Labs

wget https://dl.grafana.com/enterprise/release/grafana-enterprise-11.1.3.linux-amd64.tar.gz
tar -zxvf grafana-enterprise-11.1.0.linux-amd64.tar.gz
cd /home/grafana-11.1.0
bin/grafana-server web

2.2 容器部署

docker run -d --name=grafana -p 3000:3000 grafana/grafana-enterprise-dev:11.1.0-70874

2.3 数据源配置

http://127.0.0.1:3000

默认账号密码：admin/admin

3 node监控

3.1 node_exporter下载

wget https://github.com/prometheus/node_exporter/releases/download/v1.8.1/node_exporter-1.8.1.linux-amd64.tar.gz
tar -xvf node_exporter-1.8.1.linux-amd64.tar.gz
cd /home/ode_exporter-1.8.1.linux-amd64
./node_exporter

3.2 prometheus-job配置

scrape_configs:
  - job_name: "node"
    static_configs:
      - targets: ["9.135.95.39:9100"]

3.3 grafana监控面板

https://grafana.com/grafana/dashboards/12633-linux/

4 mysql监控

4.1 mysqld_exporter下载

wget https://github.com/prometheus/mysqld_exporter/releases/download/v0.15.1/mysqld_exporter-0.15.1.linux-amd64.tar.gz
tar -xvf mysqld_exporter-0.15.1.linux-amd64.tar.gz
cd /home/mysqld_exporter-0.15.1.linux-amd64
# 2修改配置
# cat mysql.cnf
[client]
user=root
password=admin123456
host=192.168.1.139
port=13306
# 3启动
nohup ./mysqld_exporter --web.listen-address=192.168.1.199:9104 --config.my-cnf=mysql.cnf &

4.2 prometheus-job配置

scrape_configs:
  - job_name: "mysql"
    static_configs:
      - targets: ["9.135.95.39:9104"]

4.3 grafana监控面板

https://grafana.com/grafana/dashboards/15211-mysql/

5 springboot应用监控

5.1 springboot应用配置

5.1.1 prometheus pom依赖配置

<!-- Spring Boot Actuator -->
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-actuator</artifactId>
    <version>2.7.15</version>
</dependency>
<!-- Micrometer Prometheus Registry -->
<dependency>
    <groupId>io.micrometer</groupId>
    <artifactId>micrometer-registry-prometheus</artifactId>
    <version>1.6.4</version>
</dependency>

5.1.2 application.yaml配置

spring:
  application:
    name: metrics-demo
server:
  port: 8084    

#Prometheus springboot监控配置
management:
  endpoints:
    web:
      exposure:
        include: '*'
  metrics:
    export:
      prometheus:
        enabled: true
    tags:
      application: ${spring.application.name} # 暴露的数据中添加application l

5.1.3 MeterRegistryCustomizer bean

@Bean
MeterRegistryCustomizer<MeterRegistry> configurer(
        @Value("${spring.application.name}") String applicationName) {
    return (registry) -> registry.config().commonTags("application", applicationName);
}

访问 http://9.139.95.39:8084/actuator/prometheus

5.2 prometheus-job配置

scrape_configs:
  - job_name: "metrics-demo"
    metrics_path: '/actuator/prometheus'
    static_configs:
      - targets: ["9.135.95.39:8084"]

5.3 grafana监控面板

https://grafana.com/grafana/dashboards/4701-jvm-micrometer/

5.4 PromQL 内置函数

# QPS统计
sum(rate(http_server_requests_seconds_count{application="metrics-demo"}[10s]))
# 每个接口QPS统计
rate(http_server_requests_seconds_count{application="metrics-demo"}[10s])

# 耗时统计
sum(rate(http_server_requests_seconds_sum{application="metrics-demo"}[10s])) / sum(rate(http_server_requests_seconds_count{application="metrics-demo"}[10s]))
# 每个接口的耗时
rate(http_server_requests_seconds_sum{application="metrics-demo"}[10s])

5.5 自定义指标

通过拦截器实现 MetricInterceptor.java

package com.demo.interceptor;

import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.Timer;
import org.springframework.web.servlet.ModelAndView;
import org.springframework.web.servlet.handler.HandlerInterceptorAdapter;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.time.Duration;

/**
 * 自定义指标上报
 * @author xfgeng
 * @date 2024-08-14 15:16
 */
public class MetricInterceptor extends HandlerInterceptorAdapter {

    private MeterRegistry meterRegistry;
    private ThreadLocal<Timer.Sample> threadLocal = new ThreadLocal<>();

    public MetricInterceptor(MeterRegistry meterRegistry){
        this.meterRegistry = meterRegistry;
    }

    @Override
    public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
        // 总计数 + 1
        meterRegistry.counter("api_request_count", Tags.of("url", request.getRequestURI(), "method", request.getMethod())).increment();
        // 处理中计数 +1
        meterRegistry.gauge("api_process_count", Tags.of("url", request.getRequestURI(), "method", request.getMethod()), 1);

        Timer.Sample sample = Timer.start();
        threadLocal.set(sample);

        return super.preHandle(request, response, handler);
    }

    @Override
    public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception {
        try {
            super.postHandle(request, response, handler, modelAndView);
        } finally {
            meterRegistry.gauge("api_process_count", Tags.of("url", request.getRequestURI(), "method", request.getMethod()), -1);
            //  Timer timer = meterRegistry.timer("micro_req_histogram", Tags.of("url", request.getRequestURI(), "method", request.getMethod(), "code", String.valueOf(response.getStatus())));
            Timer timer = Timer.builder("api_request_histogram").minimumExpectedValue(Duration.ofMillis(1)).maximumExpectedValue(Duration.ofMinutes(3))
                    .sla(Duration.ofMillis(10), Duration.ofMillis(50), Duration.ofMillis(100), Duration.ofMillis(300), Duration.ofMillis(1000))
                    .tags(Tags.of("url", request.getRequestURI(), "method", request.getMethod(), "code", String.valueOf(response.getStatus())))
                    .register(meterRegistry);
            threadLocal.get().stop(timer);
            threadLocal.remove();
        }
    }

}

WebMvcConfig.java

package com.demo.config;

import com.demo.interceptor.MetricInterceptor;
import io.micrometer.core.instrument.MeterRegistry;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

@Configuration
public class WebMvcConfig implements WebMvcConfigurer {

    @Autowired
    private MeterRegistry meterRegistry;

    @Override
    public void addInterceptors(InterceptorRegistry registry) {
        registry.addInterceptor(new MetricInterceptor(meterRegistry))
                .addPathPatterns("/api/**") //拦截路径
                .excludePathPatterns("/*.png","/*.gif"); //排除路径
    }

    @Bean
    MeterRegistryCustomizer<MeterRegistry> configurer(
            @Value("${spring.application.name}") String applicationName) {
        return (registry) -> registry.config().commonTags("application", applicationName);
    }
}