自定义prometheus监控获取nginx_upstream指标

1、前言

        上篇文章介绍了nginx通过nginx_upstream_check_module模块实现后端健康检查，这篇介绍一下如何自定义prometheus监控获取nginx的upstream指标来实时监控nginx。

2、nginx_upstream_status状态

支持以下三种方式查看nginx_upstream的状态

• /status?format=html

• /status?format=csv

• /status?format=json 【本文监控使用方式】

3、Python脚本自定义prometheus监控

# -*- coding:utf-8 -*-
import requests
import prometheus_client
import time
from prometheus_client import Counter, Gauge, Summary
from prometheus_client.core import CollectorRegistry
from flask import Flask, Response
from wsgiref.simple_server import make_server

app = Flask(__name__)

class Monitor:
    def __init__(self):
        self.collector_registry = CollectorRegistry(auto_describe=True)

        self.ngx_uptream_status_gauge = Gauge(name="ngx_uptream_status",
                                             documentation="nginx upstream status",
                                             labelnames=("env", "nginxUrl", "upstreamName", "upstreamAddress"),
                                             registry=self.collector_registry)

        self.ngx_status_gauge = Gauge(name="ngx_status",
                                              documentation="nginx status",
                                              labelnames=("env", "nginxUrl"),
                                              registry=self.collector_registry)

    # 获取/metrics结果
    def get_prometheus_metrics_info(self):
        return self.collector_registry

    def get_ngx_uptream_status(self):
        ngx_list = [
            {"ngx_env": "test", "ngx_status_url": "http://192.168.100.210:8888/nginx_status?format=json"}
        ]

        for ngx_info in ngx_list:
            ngxEnv = ngx_info["ngx_env"]
            ngxStatusUrl = ngx_info["ngx_status_url"]
            try:
                requests.get(ngxStatusUrl, timeout=2)
            except:
                self.ngx_status_gauge.labels(ngxEnv, "http://"+ ngxStatusUrl.split('/')[2]).set(0)
                print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " " + ngxStatusUrl + " 请求异常")
            else:
                self.ngx_status_gauge.labels(ngxEnv, "http://"+ ngxStatusUrl.split('/')[2]).set(1)
                ngxInfo = requests.get(ngxStatusUrl)
                ngxInfo.encoding = 'utf-8'
                ngxInfoJson = ngxInfo.json()
                print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " " + ngxStatusUrl + " 请求数据: " + str(ngxInfoJson))
                ngxServerList = ngxInfoJson["servers"]["server"]
                if len(ngxServerList) == 0:
                    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " " + ngxStatusUrl + " 请配置upstream检测")
                else:
                    for server in ngxServerList:
                        upstream = server["upstream"]
                        upstreamAddr = server["name"]
                        nginxUrl = "http://"+ ngxStatusUrl.split('/')[2]
                        status = server["status"]
                        if status == "up":
                            upstreamStatus = 1
                        else:
                            upstreamStatus = 0
                        self.ngx_uptream_status_gauge.labels(ngxEnv, nginxUrl, upstream, upstreamAddr).set(upstreamStatus)

@app.route('/metrics', methods=['POST', 'GET'])
def metrics():
    g_monitor = Monitor()
    g_monitor.get_ngx_uptream_status()
    registry = g_monitor.get_prometheus_metrics_info()
    return Response(prometheus_client.generate_latest(registry), mimetype="text/plain")

if __name__ == '__main__':
    server = make_server('0.0.0.0', 3200, app)
    server.serve_forever()

4、运行Python脚本，并调用接口获取metrics

Python脚本以flask服务的形式运行，通过/nginx_status?format=json接口获取到nginx_upstream的指标数据后，转换为prometheus可接受的数据形式，然后对接到prometheus即可监控。