Prometheus自定义exporter

最新推荐文章于 2024-05-31 16:49:07 发布

亭风林晚

最新推荐文章于 2024-05-31 16:49:07 发布

阅读量4.4k

点赞数

文章标签： kubernetes

本文链接：https://blog.csdn.net/weixin_46328827/article/details/106636959

版权

本文介绍了如何利用Prometheus的自定义Exporter进行GPU监控。通过参考官方文档并使用gonvml库，确保环境配置正确。文章讨论了Gauge与GaugeVec的区别，并展示了构建和测试Exporter的步骤。接着，讲述了将Exporter集成到kubernetes中，利用daemonset、service、endpoint和ServiceMonitor实现全集群监控。最后，提到了在Prometheus中查看服务发现的流程。

摘要由CSDN通过智能技术生成

服务

以GPU监控为例，参考prometheus官方文档，这里用到了gonvml库，确保运行环境能找到libnvidia-ml.so.1库，如果没有，其实也可以nvidia-smi，手动解析获取监控指标，不过这样比较麻烦。

Gauge和GaugeVec区别：Gauge单个监控指标，GaugeVec多个监控指标集合，需要定义[]labels，使用WithLabelValues(labels…)去选择指标，用于多CPU、GPU等场景

package main

import (
	"flag"
	"fmt"
	"github.com/mindprince/gonvml"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"net/http"
	"strconv"
	"sync"
)

const (
	namespace = "nvidia_gpu"
)

var (
	labels = []string{
   "minor_number", "uuid", "name"}
)

//Collector defines prometheus collector params
type Collector struct {
   
	sync.Mutex
	numDevices  prometheus.Gauge
	usedMemory  *prometheus.GaugeVec
	totalMemory *prometheus.GaugeVec
	dutyCycle   *prometheus.GaugeVec
	powerUsage  *prometheus.GaugeVec
	temperature *prometheus.GaugeVec
}

//NewCollector return a new Collector
func NewCollector() *Collector {
   
	return &Collector{
   
		numDevices: prometheus.NewGauge(
			prometheus.GaugeOpts{
   
				Namespace: namespace,
				Name:      "num_devices",
				Help:      "Number of GPU devices",
			},
		),
		usedMemory: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
   
				Namespace: namespace,
				Name:      "memory_used_bytes",
				Help:      "Memory used by the GPU device in bytes",
			},
			labels,
		),
		totalMemory: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
   
				Namespace: namespace,
				Name:      "memory_total_bytes",
				Help:      "Total memory of the GPU device in bytes",
			},
			labels,
		),
		dutyCycle: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
   
				Namespace: namespace,
				Name:      "duty_cycle",
				Help:      "Percent of time over the past sample period during which one or more kernels were executing on the GPU device",
			},
			labels,
		),
		powerUsage: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
   
				Namespace: namespace,
				Name:      "power_usage_milliwatts",
				Help:      "Power usage of the GPU device in milliwatts",
			},
			labels,
		),
		temperature: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
   
				Namespace: namespace,
				Name:      "temperature_celsius",
				Help:      "Temperature of the GPU device in celsius",
			},
			labels,
		),
	}
}

//Describe return the descriptions
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
   
	ch <- c.numDevices.Desc()
	c.usedMemory

最低0.47元/天解锁文章