这个脚本是遍历MR，如果发现这个MR是需要rebase的或者已经merge了，会自动kill相应的jenkins job。
添加crontab定时任务：*/5 * * * * python3 kill_gitlab_job.py
开启 kill rebase任务可能影响jenkins调试，谨慎开启
kill_gitlab_job.py
import re
import os
import copy
import time
import gitlab
import sqlite3
import logging
import requests
from datetime import datetime
from sqlite3 import OperationalError
from configparser import ConfigParser
from kill_jenkins_job import main
logger = logging.getLogger()
logger.setLevel(logging.WARNING) # 设置打印级别
formatter = logging.Formatter(
'%(asctime)s %(filename)s %(funcName)s [line:%(lineno)d] '
'%(levelname)s %(message)s')
# 设置屏幕打印的格式
sh = logging.StreamHandler()
sh.setFormatter(formatter)
logger.addHandler(sh)
# 设置log保存
fh = logging.FileHandler("kill_gitlab_job.log", encoding='utf8')
fh.setFormatter(formatter)
logger.addHandler(fh)
class GitlabJob(object):
def __init__(self):
cp = ConfigParser()
config_path = os.path.dirname(os.path.abspath(__file__))
cfg = os.path.join(config_path, 'config.cfg')
cp.read(cfg)
self.username = cp.get('gitlab', 'username')
self.password = cp.get('gitlab', 'password')
self.session = requests.session()
self.csrf_toke = None
self.gitlab_login()
self.gl = gitlab.Gitlab.from_config('gitlab', ['config.cfg'])
self.gl.auth()
self.job_map = {}
self.conn = sqlite3.connect('./gitlab_mr.db')
self.cur = self.conn.cursor()
self.rebase_table_name = 'gitlab_mr_rebase'
self.init_db()
self.notice = 'MESSAGE: No commit more than {} days, this merge ' \
'request will as draft'
self.mr_has_conflicts = dict()
self.mr_has_conflicts_message = '{} merge is not possible, ' \
'you can merge'
def init_db(self):
try:
sql = f"""CREATE TABLE {self.rebase_table_name} (
id integer primary key autoincrement,
`repo_id` INT NOT NULL,
`branch` VARCHAR(256) NOT NULL DEFAULT '',
`mr_iid` VARCHAR(50) NOT NULL DEFAULT ''
);"""
self.cur.execute(sql)
logging.info("create table success")
except OperationalError as o:
# logging.error(str(o))
pass
except Exception as e:
# logging.error(e)
pass
def gitlab_login(self):
index_url = 'https://gitlab.hobot.cc/'
response = self.session.get(index_url)
self.csrf_toke = re.search(r'name="csrf-token" content="(.*?)"',
response.text).group(1)
login_url = 'https://gitlab.hobot.cc/users/auth/ldapmain/callback'
data = {
'authenticity_token': self.csrf_toke,
'username': self.username,
'password': self.password,
'remember_me': '1',
}
response = self.session.post(login_url, data=data)
def handle_task(self, project_id=8189, project_setting=dict):
project = self.gl.projects.get(project_id)
if project_setting['kill_rebase_job']:
mr_list = project.mergerequests.list(all=True,
state='opened')
for mr in mr_list:
if mr.draft:
continue
self.handle_need_rebase_job(project, mr)
if project_setting['kill_merged_job']:
merged_mr_list = project.mergerequests.list(all=True,
state='merged')
ctime = datetime.now()
for mr in merged_mr_list:
try:
merged_time = datetime.strptime(
str(mr.merged_at).split(".")[0], "%Y-%m-%dT%H:%M:%S")
diff = ctime - merged_time
diff_day = int(diff.total_seconds() / 24 / 3600)
print(diff_day)
except Exception as e:
logging.error("diff time error:" + str(e))
diff_day = 2
if diff_day <= 1:
self.handle_merged_job(project, mr)
def handle_merged_job(self, project, mr):
if not mr.pipelines.list():
return
if len(mr.commits()) == 0:
return
commit_sha = [i for i in mr.commits()][0].short_id
commit = project.commits.get(commit_sha)
statuses = commit.statuses.list()
logging.info(len(statuses))
statuses_new = dict()
for sta in statuses:
statuses_new[sta.name] = sta
for name, status in statuses_new.items():
# if mr.iid == 4473:
# print(status)
if status.status == 'pending' or status.status == 'running':
logging.info(
f'{status.target_url} -- mr {mr.iid} --job {status.name}'
f' {status.status}')
self.handle_jobs(project.id, status.target_url,
status.created_at, mr.iid, status.status,
f'already merged')
def handle_need_rebase_job(self, project, mr):
if not mr.pipelines.list():
return
if len(mr.commits()) == 0:
return
widget_url = f'{mr.web_url}/widget.json'
logging.info(widget_url)
widget_response = self.session.get(widget_url)
# print(widget_response)
widget_res = widget_response.json()
if widget_res.get('should_be_rebased'):
# print(" old way to get jobs")
# latest_jobs = self.get_lasted_jobs(
# project.path_with_namespace,
# mr.pipelines.list()[0].iid)
# for job in latest_jobs:
# self.handle_jobs(project.id, job, mr.iid,
# f'should be rebased')
# print(" new way to get jobs")
commit_sha = [i for i in mr.commits()][0].short_id
commit = project.commits.get(commit_sha)
statuses = commit.statuses.list()
logging.info(len(statuses))
statuses_new = dict()
for sta in statuses:
statuses_new[sta.name] = sta
for name, status in statuses_new.items():
# if mr.iid == 4473:
# print(status)
logging.info(
f'{status.target_url} -- mr {mr.iid} --job {status.name}'
f' {status.status}')
if status.status == 'pending' or status.status == 'running':
self.handle_jobs(project.id, status.target_url,
status.created_at, mr.iid, status.status,
f'should be rebased')
@staticmethod
def time_handler(target_time: str):
local_time = datetime.strptime(target_time,
"%Y-%m-%dT%H:%M:%S.%f+08:00")
end_time = local_time.strftime("%Y-%m-%d %H:%M:%S")
time_array = time.strptime(end_time, "%Y-%m-%d %H:%M:%S")
time_stamp = int(time.mktime(time_array))
return time_stamp
# 根据mr的pipeline获取最后一组job
def get_lasted_jobs(self, namespace, iid):
url = 'https://gitlab.hobot.cc/api/graphql'
data = {
"operationName": "getPipelineJobs",
"variables": {
"fullPath": namespace,
"iid": str(iid)
},
"query": "query getPipelineJobs($fullPath: ID!, $iid: ID!, "
"$after: String) {\n project(fullPath: $fullPath) "
"{\n id\n pipeline(iid: $iid) {\n id\n "
" jobs(after: $after, first: 20) {\n pageInfo "
"{\n ...PageInfo\n __typename\n "
" }\n nodes {\n artifacts {\n "
" nodes {\n downloadPath\n "
" fileType\n __typename\n }\n"
" __typename\n }\n "
"allowFailure\n status\n scheduledAt\n "
" manualJob\n triggered\n "
"createdByTag\n detailedStatus {\n "
"id\n detailsPath\n group\n "
" icon\n label\n text\n "
" tooltip\n action {\n id\n"
" buttonTitle\n icon\n "
" method\n path\n title\n"
" __typename\n }\n "
" __typename\n }\n id\n "
"refName\n refPath\n tags\n "
"shortSha\n commitPath\n stage {\n "
" id\n name\n __typename\n "
" }\n name\n duration\n "
" finishedAt\n coverage\n retryable\n"
" playable\n cancelable\n "
"active\n stuck\n userPermissions {\n"
" readBuild\n readJobArtifacts\n "
" updateBuild\n __typename\n "
" }\n __typename\n }\n "
" __typename\n }\n __typename\n }\n "
" __typename\n }\n}\n\nfragment PageInfo on PageInfo {\n"
" hasNextPage\n hasPreviousPage\n startCursor\n "
"endCursor\n __typename\n}\n"
}
pipelines = self.session.get(url, json=data)
nodes = pipelines.json()['data']['project']['pipeline']['jobs'][
'nodes']
latest_jobs = []
for job in nodes:
latest_jobs.append(job['detailedStatus']['detailsPath'])
return latest_jobs
# 根据job链接，解析jenkins链接
def handle_jobs(self, project_id, url, created_at, mr_iid,
pipeline_job_status, message=''):
res = re.search(
r'(https://ci.*?\.hobot.cc:8443)/(.*?)/display/redirect', url)
if res:
host = res.group(1)
path = res.group(2).split('job/')
namespace = '/'.join([name.split('/')[0] for name in path])
if namespace.startswith('/'):
namespace = namespace[1:]
if not self.job_map.get(namespace):
self.job_map[namespace] = {'host': host,
'project_id': project_id,
'namespace': namespace,
'item': {}}
logging.info(
f'project {project_id} -- mr {mr_iid} --job {namespace} '
f'{message}')
# self.kill_job_with_mr_iid(mr_iid, url)
temp = dict()
temp['pipeline_job_status'] = pipeline_job_status
temp['pipeline_job_created_at'] = created_at
temp['message'] = message
self.job_map[namespace]['item'][mr_iid] = temp
# 发送评论
@staticmethod
def send_gitlab_note(project, mr, note):
try:
mr = project.mergerequests.get(mr.iid)
mr.notes.create({'body': note})
logging.info(f'{mr.iid} --{mr.web_url} ---{note}')
except Exception as e:
logging.error("send gitlab note failed:" + str(e))
def notice_mr(self, project, mr, project_setting: dict):
auto_draft = project_setting.get('auto_draft', False)
send_frequency = project_setting.get('send_frequency', 2)
send_times_to_draft = project_setting.get('send_times_to_draft', 4)
print(send_frequency)
print(send_times_to_draft)
num = 0
notice_list = mr.notes.list()
for index, attr in enumerate(notice_list):
if attr.body.startswith('MESSAGE: No commit more than'):
num += 1
# 超过两天没有评论，给他提醒
if index == 0 and time.time() - self.time_handler(
attr.updated_at) > 3600 * 24 * send_frequency:
notice = self.notice.format((index + 1) * send_frequency)
self.send_gitlab_note(project, mr, notice)
num += 1
if index >= send_times_to_draft:
break
if num >= send_times_to_draft and auto_draft:
try:
self.set_draft_to_mr(project, mr)
except Exception as e:
logging.error("set mr draft with error:" + str(e))
# 将mr设置为draft: 只要将mr的title开头加上 "Draft:" 这个mr将自动设置为draft
@staticmethod
def set_draft_to_mr(project, mr):
if not mr.draft:
title = mr.title
mr.title = 'Draft: ' + title
mr.save()
mr = project.mergerequests.get(mr.iid)
if mr.draft:
logging.info(f'set:{project.id} {mr.iid} draft succeed!')
else:
logging.error(f'set:{project.id} {mr.iid} draft failed!')
def kill_job_with_mr_iid(self):
for namespace, job in self.job_map.items():
print(job['item'])
host = job['host']
mr_pipeline = job['item']
project_id = job['project_id']
logging.debug(job)
main(host, namespace, project_id, mr_pipeline)
@staticmethod
def handle_message(reason, link):
return f"""Build Result: ABORT\n
REASON: {reason}\n
Job Link: {link}\n
"""
if __name__ == '__main__':
"""
kill_rebase_job: 是否检查并kill需要rebase的job, 默认 False
kill_merged_job: 是否检查并kill已经merge的mr正在跑的job，默认False
"""
repo_tasks = {
7484: {
'kill_rebase_job': False,
'kill_merged_job': True,
},
# 8189: {
# 'kill_rebase_job': False,
# 'kill_merged_job': True,
# }
}
for repo, repo_task_set in repo_tasks.items():
g = GitlabJob()
g.handle_task(repo, repo_task_set)
logging.info(g.job_map)
g.kill_job_with_mr_iid()
kill_jenkins_job.py
import os
import re
import time
import gitlab
import logging
import json
import jenkins
import requests
from lxml import etree
from configparser import ConfigParser
logger = logging.getLogger()
logger.setLevel(logging.WARNING) # 设置打印级别
formatter = logging.Formatter(
'%(asctime)s %(filename)s %(funcName)s [line:%(lineno)d] '
'%(levelname)s %(message)s')
# 设置屏幕打印的格式
sh = logging.StreamHandler()
sh.setFormatter(formatter)
logger.addHandler(sh)
# 设置log保存
fh = logging.FileHandler("kill_gitlab_job.log", encoding='utf8')
fh.setFormatter(formatter)
logger.addHandler(fh)
class JenkinsStopTask(object):
def __init__(self, host, user, passwd, namespace, project_id, mr_ids):
self.host = host
self.user = user
self.password = passwd
self.server = None
self.init_server()
self.namespace = namespace
self.mr_ids = mr_ids
self.project_id = project_id
self.need_send_note_job = list()
self.gl = gitlab.Gitlab.from_config('gitlab', ['config.cfg'])
self.gl.auth()
self.gitlab_job_name_map = {
"HAT/HAT": "Jenkins",
"HAT/Projects/Pilot": "Pilot-Jenkins",
"HAT/Projects/SP": "SP-Jenkins",
"auto-perception/HDFlow": "Jenkins",
"devops_se_scm/MR_LINT": "MR_LINT",
"devops_se_scm/MR_LINT_TEST": "MR_LINT_TEST",
"devops/ci_pipeline": "ci_pipeline"
}
def init_server(self):
self.server = jenkins.Jenkins(self.host,
username=self.user,
password=self.password)
def get_running_build(self, job):
tmp_url = ''
if isinstance(job, str):
job = job.split('/')
for jj in job:
tmp_url += f'/job/{jj}'
url = f'{self.host}{tmp_url}'
api_url = f'{self.host}{tmp_url}/api/json'
response = self.server.jenkins_open(
requests.Request('GET', api_url))
response_json = json.loads(response)
if response_json[
'_class'] == 'org.jenkinsci.plugins.workflow.job.WorkflowJob':
response = self.server.jenkins_open(
requests.Request('GET', url))
element = etree.HTML(response)
self.parse_job(element, response_json['fullName'])
else:
for job in response_json['jobs']:
if job['_class'] == \
'org.jenkinsci.plugins.workflow.job.WorkflowJob':
sub_job_response = self.server.jenkins_open(
requests.Request('GET', job['url']))
sub_job_element = etree.HTML(sub_job_response)
tmp_url1 = job['url']
if tmp_url1.endswith('/'):
tmp_url1 = tmp_url1[:-1]
sub_full_name = response_json['fullName'] + '/' + \
tmp_url1.split('/')[-1]
self.parse_job(sub_job_element, sub_full_name)
def parse_job(self, element, job):
task_list = element.xpath('//*[@id="buildHistory"]/div[2]/table/tr/td')
for task in task_list:
res = task.xpath('div[1]/div/a/span/svg/@tooltip')
if res and 'In progress' in res[0]:
try:
task_name = None
# merge requests
if task.xpath('div[4]/a/text()'):
merge_text = task.xpath('div[4]/a/text()')[0]
if merge_text and 'GitLab Merge Request' in merge_text:
task_name_array = task.xpath('./div[4]/a/@href')
if task_name_array:
task_name = task_name_array[0]
# 不是这两项，不处理
if not task_name:
continue
mr_iid = task_name.split('/')[-1]
if int(mr_iid) not in self.mr_ids.keys():
continue
index = None
index_array = task.xpath('./div[1]/a/text()')
if index_array:
index_str = index_array[0]
index = re.search(r'(\d+)', index_str).group(1)
if index:
job_url = task.xpath('./div[1]/a/@href')[0]
# notice = f'kill {self.host}{job_url}, because ' \
# f'{self.mr_ids[int(mr_iid)]}'
message = self.mr_ids[int(mr_iid)]['message']
notice = self.handle_message(
f'{message}',
f'{self.host}{job_url}')
self.stop_task(f'{self.host}{job_url}', job, index,
mr_iid, notice, message)
except Exception as err:
logger.error(err)
def get_request_json(self, url):
response = self.server.jenkins_open(
requests.Request('GET', url))
json_result = json.loads(response)
return json_result
def stop_task(self, job_link, job, index, mr_iid, note, message):
if isinstance(job, list):
job = '/'.join(job)
logging.warning(f"{self.namespace}---{job} --- {index}--- {note}")
# 正式环境放开下边两项
self.server.stop_build(job, index)
if message == 'should be rebased':
self.send_stop_task_note(mr_iid, note)
# result = self.get_request_json(job_link)
# print(result['lastBuiltRevision'])
# self.need_send_note_job.append(job_link)
# job_link = job_link + 'api/json?pretty=true'
# for i in range(5):
# time.sleep(1)
# result = self.get_request_json(job_link)
# print(result['building'])
# if not result['building']:
# self.send_stop_task_note(mr_iid, note)
# break
def send_stop_task_note(self, mr_iid, note):
project = self.gl.projects.get(self.project_id)
mr = project.mergerequests.get(mr_iid)
mr.notes.create({'body': note})
if self.mr_ids[int(mr_iid)]['pipeline_job_status'] == 'pending':
commit_sha = [i for i in mr.commits()][0].short_id
commit = project.commits.get(commit_sha)
statuses = commit.statuses.list()
logging.debug("get statues")
if self.namespace in self.gitlab_job_name_map.keys():
gitlab_job_name = self.gitlab_job_name_map[self.namespace]
for status in statuses:
logging.debug(status)
if status.name == gitlab_job_name:
commit.statuses.create({
'state': 'canceled',
'description': 'canceled',
'name': gitlab_job_name,
'target_url': status.target_url
})
else:
pass
@staticmethod
def handle_message(reason, link):
return f"""Build Result: ABORT\n
REASON: {reason}\n
Job Link: {link}\n
"""
def handle(self):
for _ in range(1):
self.get_running_build(self.namespace)
def main(host, namespace, project_id, mr_pipelines):
cp = ConfigParser()
config_path = os.path.dirname(os.path.abspath(__file__))
cfg = os.path.join(config_path, 'config.cfg')
cp.read(cfg)
username = cp.get('jenkins', 'username')
password = cp.get('jenkins', 'password')
jen = JenkinsStopTask(host, username, password, namespace, project_id,
mr_pipelines)
jen.handle()
config.cfg
[jenkins]
username =
password =
[gitlab]
Username =
password =
url = https://gitlab.hobot.cc
private_token =
api_version = 4
扫一扫
1029
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
