需求
1.jenkins在应用发版过程中的java进程重启,会导致监控系统报警,此类发版过程报警运维人员可忽略;
2.jenkins在应用在发版过程中的java进程重启,nginx代理会进行摘节点操作,保证服务对外无影响;
3.zabbix监控系统需求对nginx的摘节点行为进行监控(发版过程中的摘节点不进行报警);
4.对nginx摘节点行为进行excel记录,记录time,upstream,name
为保证监控系统检查到切实有效的报警信息,我们需要对监控进行优化。
思路
1.通过nginx的后端节点状态页面,获取当前nginx中有哪些后端节点;
2.调用jenkins api获取jenkins当前处于构建状态的列表,若running列表中没有对应的构建任务,则不是发版导致摘节点,则报警并记录到excel中;若有,则说明是jenkins发版导致的摘节点,不进行报警。
实现
1.python 脚本
#!/usr/bin/bin python
#-*- coding: UTF-8 -*-
'''
auth:yanggd
date:2019-03-25
comment:
1.结合zabbix监控内网nginx的节点状态,若为down则报警
2.生成摘节点报表,记录time,upstream,name
3.安装python-jenkins,requests,xlwt,xlrd,xlutils
4.使用jenkins api排查发版过程中的摘节点
'''
import jenkins
import json
import requests
import datetime
#写excel
import xlwt
#读excel
import xlrd
import os
import sys
from xlutils.copy import copy
#连接jenkins api
def getJenkins():
jenkins_url = 'http://jenkins.test.cn'
username = 'admin'
password = '12345678'
server = jenkins.Jenkins(jenkins_url, username, password)
return server
#新建xls并添加数据
def writeExcel(down_list, file):
header = ['time', 'upstream', 'name']
workbook = xlwt.Workbook()
worksheet = workbook.add_sheet('nginx_upstream_status')
#写表头
i = 0
for each_header in header:
worksheet.write(0, i, each_header)
i += 1
#写内容
row = 1
for each_row in down_list:
col = 0
for each_col in header:
worksheet.write(row, col, each_row[each_col])
col += 1
row += 1
workbook.save(file)
#追加xls数据
def writeExcel_add(down_list, file):
header = ['time', 'upstream', 'name']
workbook = xlrd.open_workbook(file)
worksheet = workbook.sheet_by_name('nginx_upstream_status')
rows_old = worksheet.nrows
new_workbook = copy(workbook)
new_worksheet = new_workbook.get_sheet(0)
row = 0
for each_row in down_list:
col = 0
for each_col in header:
new_worksheet.write(row+rows_old, col, each_row[each_col])
col += 1
row += 1
new_workbook.save(file)
#自动发现
def nodeDiscovery():
node_list = []
#nignx upstream 状态监控页,注意修改此url,一共有两处
upstream_url = 'http://192.168.3.141/upstream_status?format=json'
res = requests.get(upstream_url)
for node in res.json()['servers']['server']:
node_dic = {}
node_dic['{#UPSTREAM}'] = node['upstream']
node_dic['{#NAME}'] = node['name']
#node_dic['{#STATUS}'] = node['status']
node_list.append(node_dic)
node_info = {"data":node_list}
print json.dumps(node_info,sort_keys=True, indent=4, separators=(', ', ': '))
#节点状态
def nodeStatus(upstream, name):
#报表名
mulu = '/App/zabbix/etc/zabbix_agentd.conf.d'
xls = 'upstream_status.xls'
#新建字典,用于映射nginx upstream和jenkins 项目的映射关系
upstream_dic = {
"test1" : "prod-test1",
"test2" : "prod-test2",
}
#空字典,用于存储down节点信息
down_dic = {}
#空列表,用于存储多个down节点
down_list = []
#获取时间
time = datetime.datetime.now()
#nignx upstream 状态监控页
upstream_url = 'http://192.168.3.141/upstream_status?format=json'
res = requests.get(upstream_url)
num = 0
for node in res.json()['servers']['server']:
if node['name'] == name and node['status'] == 'down':
#检查jenkins是否正在发版
server = getJenkins()
#获取jenkins处于构建状态的列表
running_list = server.get_running_builds()
#判断running_list是否为空
count = 0
if len(running_list):
print name + "test"
for item in running_list:
if upstream_dic[upstream] == item['name']:
count += 1
#jenkins running列表中没有对应的构建任务,则不是发版导致摘节点
if count == 0:
#节点down,打印1
print 1
#添加down节点信息
down_dic['time'] = str(time)
down_dic['upstream'] = upstream
down_dic['name'] = name
#添加down节点
down_list.append(down_dic)
file = os.path.join(mulu, xls)
#若文件不存在则创建文件,若存在则追加内容
if os.path.exists(file):
writeExcel_add(down_list, file)
else:
writeExcel(down_list, file)
#for down_item in down_list:
# print down_item['time'] + down_item['upstream'] + down_item['name']
else:
#节点down,打印1
print 1
#添加down节点信息
down_dic['time'] = str(time)
down_dic['upstream'] = upstream
down_dic['name'] = name
#添加down节点
down_list.append(down_dic)
file = os.path.join(mulu, xls)
#若文件不存在则创建文件,若存在则追加内容
if os.path.exists(file):
writeExcel_add(down_list, file)
else:
writeExcel(down_list, file)
#for down_item in down_list:
# print down_item['time'] + down_item['upstream'] + down_item['name']
num += 1
if num == 0:
#节点没有down,打印2
print 2
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: python " + sys.argv[0] + ' [nodeDiscovery|nodeStatus]'
else:
if sys.argv[1] == 'nodeDiscovery':
nodeDiscovery()
elif sys.argv[1] == 'nodeStatus':
if len(sys.argv) != 4:
print "Usage: python " + sys.argv[0] + ' nodeStatus upstream name status'
else:
upstream = sys.argv[2]
name = sys.argv[3]
nodeStatus(upstream, name)
else:
print "Usage: python " + sys.argv[0] + ' ' + '[nodeDiscovery|nodeStatus]'
2.zabbix监控设置
#nginx upstream status监控
UserParameter=upstream.discovery,python /App/zabbix/etc/zabbix_agentd.conf.d/upstream_status.py nodeDiscovery
UserParameter=upstream.status[*],python /App/zabbix/etc/zabbix_agentd.conf.d/upstream_status.py nodeStatus '$1' '$2'