zabbix实现nginx的摘节点状态监控

需求

1.jenkins在应用发版过程中的java进程重启，会导致监控系统报警，此类发版过程报警运维人员可忽略；
2.jenkins在应用在发版过程中的java进程重启，nginx代理会进行摘节点操作，保证服务对外无影响；
3.zabbix监控系统需求对nginx的摘节点行为进行监控（发版过程中的摘节点不进行报警）；
4.对nginx摘节点行为进行excel记录，记录time,upstream,name

为保证监控系统检查到切实有效的报警信息，我们需要对监控进行优化。

思路

1.通过nginx的后端节点状态页面，获取当前nginx中有哪些后端节点；
2.调用jenkins api获取jenkins当前处于构建状态的列表，若running列表中没有对应的构建任务，则不是发版导致摘节点，则报警并记录到excel中；若有，则说明是jenkins发版导致的摘节点，不进行报警。

实现

1.python 脚本

#!/usr/bin/bin python
#-*- coding: UTF-8 -*-
'''
auth:yanggd
date:2019-03-25
comment:
1.结合zabbix监控内网nginx的节点状态,若为down则报警
2.生成摘节点报表,记录time,upstream,name
3.安装python-jenkins,requests,xlwt,xlrd,xlutils
4.使用jenkins api排查发版过程中的摘节点
'''

import jenkins
import json
import requests
import datetime
#写excel
import xlwt
#读excel
import xlrd
import os
import sys
from xlutils.copy import copy

#连接jenkins api
def getJenkins():
        jenkins_url = 'http://jenkins.test.cn'
        username = 'admin'
        password = '12345678'
        server = jenkins.Jenkins(jenkins_url, username, password)
        return server
#新建xls并添加数据
def writeExcel(down_list, file):
        header = ['time', 'upstream', 'name']
        workbook = xlwt.Workbook()
        worksheet = workbook.add_sheet('nginx_upstream_status')
        #写表头
        i = 0
        for each_header in header:
                worksheet.write(0, i, each_header)
                i += 1
        #写内容
        row = 1
        for each_row in down_list:
                col = 0
                for each_col in header:
                        worksheet.write(row, col, each_row[each_col])
                        col += 1
                row += 1
        workbook.save(file)

#追加xls数据
def writeExcel_add(down_list, file):
        header = ['time', 'upstream', 'name']
        workbook = xlrd.open_workbook(file)
        worksheet = workbook.sheet_by_name('nginx_upstream_status')
        rows_old = worksheet.nrows
        new_workbook = copy(workbook)
        new_worksheet = new_workbook.get_sheet(0)

        row = 0
        for each_row in down_list:
                col = 0
                for each_col in header:
                        new_worksheet.write(row+rows_old, col, each_row[each_col])
                        col += 1
                row += 1
        new_workbook.save(file)
#自动发现
def nodeDiscovery():
        node_list = []
        #nignx upstream 状态监控页,注意修改此url，一共有两处
        upstream_url = 'http://192.168.3.141/upstream_status?format=json'
        res = requests.get(upstream_url)
        for node in res.json()['servers']['server']:
                node_dic = {}
                node_dic['{#UPSTREAM}'] = node['upstream']
                node_dic['{#NAME}'] = node['name']
                #node_dic['{#STATUS}'] = node['status']
                node_list.append(node_dic)

        node_info = {"data":node_list}
        print json.dumps(node_info,sort_keys=True, indent=4, separators=(', ', ': '))

#节点状态
def nodeStatus(upstream, name):
        #报表名
        mulu = '/App/zabbix/etc/zabbix_agentd.conf.d'
        xls = 'upstream_status.xls'
        #新建字典，用于映射nginx upstream和jenkins 项目的映射关系
        upstream_dic = {
                "test1" : "prod-test1",
                "test2" : "prod-test2",
        }
        #空字典，用于存储down节点信息
        down_dic = {}
        #空列表，用于存储多个down节点
        down_list = []
        #获取时间
        time = datetime.datetime.now()
        #nignx upstream 状态监控页
        upstream_url = 'http://192.168.3.141/upstream_status?format=json'
        res = requests.get(upstream_url)
        num = 0
        for node in res.json()['servers']['server']:
                if node['name'] == name and node['status'] == 'down':
                        #检查jenkins是否正在发版
                        server = getJenkins()
                        #获取jenkins处于构建状态的列表
                        running_list = server.get_running_builds()
                        #判断running_list是否为空
                        count = 0
                        if len(running_list):
                                print name + "test"
                                for item in running_list:
                                        if upstream_dic[upstream] == item['name']:
                                                count += 1
                                #jenkins running列表中没有对应的构建任务，则不是发版导致摘节点
                                if count == 0:
                                        #节点down，打印1
                                        print 1
                                        #添加down节点信息
                                        down_dic['time'] = str(time)
                                        down_dic['upstream'] = upstream
                                        down_dic['name'] = name
                                        #添加down节点
                                        down_list.append(down_dic)
                                        file = os.path.join(mulu, xls)
                                        #若文件不存在则创建文件，若存在则追加内容
                                        if os.path.exists(file):
                                                writeExcel_add(down_list, file)
                                        else:
                                                writeExcel(down_list, file)
                                        #for down_item in down_list:
                                        #       print down_item['time'] + down_item['upstream'] + down_item['name']
                        else:
                                #节点down，打印1
                                print 1
                                #添加down节点信息
                                down_dic['time'] = str(time)
                                down_dic['upstream'] = upstream
                                down_dic['name'] = name
                                #添加down节点
                                down_list.append(down_dic)

                                file = os.path.join(mulu, xls)
                                #若文件不存在则创建文件，若存在则追加内容
                                if os.path.exists(file):
                                        writeExcel_add(down_list, file)
                                else:
                                        writeExcel(down_list, file)
                                #for down_item in down_list:
                                #       print down_item['time'] + down_item['upstream'] + down_item['name']
                        num += 1
        if num == 0:
                #节点没有down，打印2
                print 2


if __name__ == '__main__':
        if len(sys.argv) < 2:
                print "Usage: python " + sys.argv[0] + ' [nodeDiscovery|nodeStatus]'
        else:
                if sys.argv[1] == 'nodeDiscovery':
                        nodeDiscovery()
                elif sys.argv[1] == 'nodeStatus':
                        if len(sys.argv) != 4:
                                print "Usage: python " + sys.argv[0] + '  nodeStatus upstream name status'
                        else:
                                upstream = sys.argv[2]
                                name = sys.argv[3]

                                nodeStatus(upstream, name)

                else:
                        print "Usage: python " + sys.argv[0] + ' ' + '[nodeDiscovery|nodeStatus]'

2.zabbix监控设置

#nginx upstream status监控
UserParameter=upstream.discovery,python /App/zabbix/etc/zabbix_agentd.conf.d/upstream_status.py nodeDiscovery
UserParameter=upstream.status[*],python /App/zabbix/etc/zabbix_agentd.conf.d/upstream_status.py nodeStatus '$1' '$2'