k8s sidecar接入prometheus+grafana

1.首先需要写一个prometheus需要采集的接口。

func (logcar *LogCar) runHttp() {
	prometheus.Register(logcar.exporter)
	http.Handle("/metrics", promhttp.Handler())
	if err := http.ListenAndServe(":"+strconv.Itoa(utils.SIDECAR_METRICS), nil); err != nil {
		fmt.Println("Error occur when start server", err)
		return
	}
}

2.编写需要用到的各种指标。

type Exporter struct {
	mux          *http.ServeMux
	mu           *sync.RWMutex
	requestCount int64

	goroutinesDesc *prometheus.Desc //协程监控
	requestDesc    *prometheus.Desc //请求次数监控
	requestSummary *prometheus.Desc //请求相应时间监控
	memoryDesc     statsMetrics     //内存监控
	cpuRateDesc    *prometheus.Desc //cpu使用率监控
	cpuLoad1Desc   *prometheus.Desc //cpu负载监控
	cpuLoad5Desc   *prometheus.Desc //cpu负载监控
	cpuLoad15Desc  *prometheus.Desc //cpu负载监控
}

3.实现prometheus需要实现的接口。

func NewExporter() *Exporter {
	mux := http.NewServeMux()
	e := &Exporter{
		mux:            mux,
		mu:             &sync.RWMutex{},
		requestCount:   0,
		goroutinesDesc: prometheus.NewDesc("goroutine_count", "协程数量", nil, nil),
		requestDesc:    prometheus.NewDesc("request_count", "请求数量", nil, nil),
		requestSummary: prometheus.NewDesc("request_summary", "请求数量", nil, nil),
		memoryDesc: statsMetrics{
			{
				desc: prometheus.NewDesc(
					"total_mem",
					"内存总量",
					nil, nil),
				valType: prometheus.GaugeValue,
				eval:    func(ms *mem.VirtualMemoryStat) float64 { return float64(ms.Total) / 1e9 },
			},
			{
				desc: prometheus.NewDesc(
					"free_mem",
					"内存空闲",
					nil, nil),
				valType: prometheus.GaugeValue,
				eval:    func(ms *mem.VirtualMemoryStat) float64 { return float64(ms.Free) / 1e9 },
			},
		},
		cpuRateDesc:   prometheus.NewDesc("cpu_rate", "cpu使用率", nil, nil),
		cpuLoad1Desc:  prometheus.NewDesc("cpu_load1", "cpu1分钟负载", nil, nil),
		cpuLoad5Desc:  prometheus.NewDesc("cpu_load5", "cpu5分钟负载", nil, nil),
		cpuLoad15Desc: prometheus.NewDesc("cpu_load15", "cpu15分钟负载", nil, nil),
		//diskIODesc:prometheus.NewDesc("disk_io","磁盘io", nil,nil),
		//diskRateDesc:prometheus.NewDesc("disk_rate","磁盘使用率", nil,nil),
	}

	return e
}

func (n *Exporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- n.goroutinesDesc
	ch <- n.requestDesc
	ch <- n.requestSummary
	for _, metric := range n.memoryDesc {
		ch <- metric.desc
	}
	ch <- n.cpuRateDesc
	ch <- n.cpuLoad1Desc
	ch <- n.cpuLoad5Desc
	ch <- n.cpuLoad15Desc
	//ch <- n.diskIODesc
	//ch <- n.diskRateDesc
}

func (n *Exporter) Collect(ch chan<- prometheus.Metric) {
	n.mu.RLock()
	defer n.mu.RUnlock()
	ch <- prometheus.MustNewConstMetric(n.goroutinesDesc, prometheus.GaugeValue, float64(runtime.NumGoroutine()))
	ch <- prometheus.MustNewConstMetric(n.requestDesc, prometheus.GaugeValue, float64(n.requestCount))
	//request后期想办法修改
	ch <- prometheus.MustNewConstMetric(n.requestSummary, prometheus.GaugeValue, float64(n.requestCount))
	//内存使用
	vm, err := mem.VirtualMemory()
	if err != nil {
		fmt.Println(err)
		return
	}
	for _, metric := range n.memoryDesc {
		ch <- prometheus.MustNewConstMetric(metric.desc, metric.valType, metric.eval(vm))
	}
	//cpu使用率
	cpuRate, err := cpu.Percent(time.Second*2, false)
	if err != nil {
		fmt.Println(err)
		return
	}
	ch <- prometheus.MustNewConstMetric(n.cpuRateDesc, prometheus.GaugeValue, cpuRate[0])

	//cou负载
	info, err := load.Avg()
	if err != nil {
		fmt.Println(err)
		return
	}
	ch <- prometheus.MustNewConstMetric(n.cpuLoad1Desc, prometheus.GaugeValue, info.Load1)
	ch <- prometheus.MustNewConstMetric(n.cpuLoad5Desc, prometheus.GaugeValue, info.Load5)
	ch <- prometheus.MustNewConstMetric(n.cpuLoad15Desc, prometheus.GaugeValue, info.Load15)
}

3.在sidecar注入的时候需要给pod加一些annotation，来让prometheus识别该pod是需要进行数据采集的。

func Annotation(obj runtime.Object) {
	m := map[string]string{
		"prometheus.io/scrape": "true",
		"prometheus.io/path":   "/metrics",
		"prometheus.io/port":   strconv.Itoa(SIDECAR_METRICS),
	}
	switch obj.(type) {
	case *v1.Pod:
		pod := obj.(*v1.Pod)
		if len(pod.Annotations) > 0 {
			for k, v := range m {
				pod.Annotations[k] = v
			}
		} else {
			pod.Annotations = m
		}
	case *v12.Deployment:
		deploy := obj.(*v12.Deployment)
		if len(deploy.Spec.Template.Annotations) > 0 {
			for k, v := range m {
				deploy.Spec.Template.Annotations[k] = v
			}
		} else {
			deploy.Spec.Template.Annotations = m
		}
	case *v13.Job:
		job := obj.(*v13.Job)
		if len(job.Spec.Template.Annotations) > 0 {
			for k, v := range m {
				job.Spec.Template.Annotations[k] = v
			}
		} else {
			job.Spec.Template.Annotations = m
		}
	}
}

4.就可以在grafana中进行绘制图片了。

logcar: k8s sidecar