check build status

# !/usr/bin/env python
# encoding:utf-8

# from selenium import webdriver
import time
import re
import json
from bs4 import BeautifulSoup
import requests
import pymsteams


def _get_table():
    while True:
        try:
            html = requests.get('https://builds.wgti.net', verify=False)
            body = html.text
            if html.status_code == 200:
                break
        except Exception as e:
            print e
            time.sleep(60)

    soup = BeautifulSoup(body, 'lxml')
    tables = soup.findAll('table')
    table2 = tables[1]
    return table2


def _get_rows():
    content = ''
    row = []
    table = _get_table()
    for tr in table.findAll('tr'):
        for td in tr.findAll('td'):
            # print td.getText()
            content = content + td.getText() + ' '
        row.append(content)
        content = ''
    row.pop(0)
    row.pop()

    new_rows = []
    for item in row:
        new_rows.append(item.strip())
    print new_rows
    return new_rows


def _get_date(row_list):
    date = []
    for item in row_list:
        # print item
        dt = re.search(r'\d+/\d+/\d+ \d+:\d+', item).group(0)
        date.append(dt)
    return date


def _get_arr_no_date(row_list):
    new_arr = []
    for item in row_list:
        new_item = re.sub('\d+/\d+/\d+ \d+:\d+', '', item)
        new_arr.append(new_item)
    return new_arr


def _get_project(row_list):
    project = []
    new_arr_list = _get_arr_no_date(row_list)
    for item in new_arr_list:
        project.append(item.split()[0])
    return project


def _get_status(row_list):
    status = []
    new_arr_list = _get_arr_no_date(row_list)
    for item in new_arr_list:
        status.append(item.split()[1])
    return status


def _get_build(row_list):
    build = []
    new_arr_list = _get_arr_no_date(row_list)
    for item in new_arr_list:
        build.append(item.split()[2])
    return build


def check_status(sta):
    if sta == 'Idle':
        return True
    else:
        return False


def _post_data(data):
    data = json.dumps(data)
    new_data = re.sub('"true"', 'true', data)
    print new_data
    # response = requests.post('http://watsng-staging.wgti.net/job/', new_data)
    # response = requests.post('http://10.138.110.92/job/', new_data)
    response = requests.post('http://web/job/', new_data)  # inside url in docker
    print response.text


def send_job_request(job_list):
    data = dict(library_branch='dev', suite_name='nightly-regression',
                build='', branch='', submitter='wats-ng@watchguard.com',
                script_branch='dev', private_rig='RIG-Nightly-regression',
                nightly_build='true')
    if job_list == success:
        for item in job_list:
            branch = item.split()[0]
            build = item.split()[2]
            if branch in white_list:
                data['suite_name'] = 'Nightly-' + branch
                data['branch'] = branch
                data['build'] = build
                _post_data(data)
                time.sleep(3)
                print 'Success job request send completed!'
            else:
                print 'branch: %s is success, but it is not in white list, do NOT send success job request.' % branch
    else:
        for item in job_list:
            if item == 'utm-inc':
                print 'build: \'utm-inc\' is failed, but do NOT send fail job request.'
            else:
                data['suite_name'] = 'Nightly-' + item
                data['branch'] = item
                data['build'] = 'failed'
                _post_data(data)
                time.sleep(3)
                print 'Fail job request send completed!'


def get_diff_indx(list1, list2):
    id = []
    for i in xrange(len(list1)):
        item_a = list1[i]
        item_b = list2[i]
        if item_a != item_b:
            id.append(i)
    return id


def get_equal_index(list1, list2):
    id = []
    for i in xrange(len(list1)):
        item_a = list1[i]
        item_b = list2[i]
        if item_a == item_b:
            id.append(i)
    return id


def get_unfinished_and_failed(old, new, content):
    diff_id = get_diff_indx(old, new)
    equal_id = get_equal_index(old, new)
    for i in diff_id:
        if old[i] != 'Idle' and new[i] == 'Idle':
            fail.append(content[i])   # send notification-------------->>>>>
            reason = 'This branch: %s\'s status change from \'non-Idle\' to \'Idle\' ' \
                     'but there is no new image generation' % content[i]
            fail_dict['fail_reason'] = reason
            fail_dict['Branch'] = content[i]
            fail_dict_list.append(fail_dict)
        else:
            unfinished.append(content[i])
    for i in equal_id:
        if old[i] == 'Idle' and new[i] == 'Idle':
            pass
        else:
            unfinished.append(content[i])

    return unfinished, fail_dict_list


def get_unfinished_success_fail(idx, status_list, arr_list, record_unfinished):
    success_branch = []
    for i in xrange(idx):
        if check_status(status_list[i]):
            success.append(arr_list[i])
            success_branch.append(arr_list[i].split()[0])
        # else:
        #     send_notification('Unknown error occurred!')   # send notification-------------->>>>>
    for m in range(idx, len(status_list)):
        project_name = arr_list[m].split()[0]
        if not status_list[m] == 'Idle':
            unfinished.append(project_name)
    for un_job in record_unfinished:
        if un_job not in unfinished and un_job not in success_branch:
            fail.append(un_job)   # send notification-------------->>>>>
            reason = 'This project: %s build failed when some other branch build success' % un_job
            fail_dict['fail_reason'] = reason
            fail_dict['Branch'] = un_job
            fail_dict_list.append(fail_dict)

    return unfinished, success, fail_dict_list


def init_timer_list(n):
    timer = []
    for i in xrange(n):
        # timer.append(0)
        timer.append(100)     # testing
    return timer


def _send_hipchat_msg(msg, good=True):
    url = '%s/v2/room/%s/notification?auth_token=%s' % (HIPCHAT_SERVER, HIPCHAT_ROOM, HIPCHAT_TOKEN)
    headers = {'content-type': 'application/json'}
    if good:
        data = {'message': msg, 'from': 'West', 'color': 'green', 'style': 'application'}
    else:
        data = {'message': msg, 'from': 'West', 'color': 'red', 'style': 'application'}
        print 'Something went wrong, send message to hipchat, message: %s' % msg
    resp = requests.post(url, data=json.dumps(data), verify=False, headers=headers)
    if resp.status_code == 204:
        print 'Send notification succeed'
    else:
        print 'Send notification failed caused by %s', resp.text


def send_success_notification():
    for item in success_dict_list:
        branch = item['branch']
        build = item['build']
        if 'error_info' in item.keys():
            error_info = item['error_info']
            msg = 'Branch          : %s\n\r' \
                  'Build           : %s\n\r' \
                  'Status          : Build Success\n\r' \
                  'Error_info      : %s' % (branch, build, error_info)
        else:
            start_time = item['start_time']
            finish_time = item['finish_time']
            msg = 'Branch          : %s\n\r' \
                  'Build           : %s\n\r' \
                  'Status          : Build Success\n\r' \
                  'Start Time      : %s\n\r' \
                  'Finish Time     : %s' \
                  % (branch, build, start_time, finish_time)

        if branch in white_list:
            msg = msg + '\n\rWhite List      : Yes' \
                        '\n\rSend Job Request: Yes'
        else:
            msg = msg + '\n\rWhite List      : No' \
                        '\n\rSend Job Request: No'

        _send_hipchat_msg(msg)
        print 'hipchat notification send completed!'
        _send_msteams_msg(msg)
        print 'msteams notification send completed!'


def send_fail_notification():
    for item in fail_dict_list:
        branch = item['Branch']
        reason = fail_dict['fail_reason']
        msg = 'Branch          : %s\n\r' \
              'Status          : Build Failed\n\r' \
              'Fail Reason     : %s' % (branch, reason)
        if branch in white_list:
            msg = msg + '\n\rWhite List      : Yes' \
                        '\n\rSend Job Request: Yes'
        else:
            msg = msg + '\n\rWhite List      : No' \
                        '\n\rSend Job Request: Yes'

        _send_hipchat_msg(msg, good=False)
        print 'hipchat_notification send completed!'
        _send_msteams_msg(msg)
        print 'msteams_notification send completed!'


def send_messages(msg):
    _send_hipchat_msg(msg)
    _send_msteams_msg(msg)


def _send_msteams_msg(msg, good=True):
    myteamsmessage = pymsteams.connectorcard(
        'https://outlook.office.com/webhook/'
        '0f7a6329-9345-4c6e-a313-dc7a48cfdd6b@2563c132-88f5-466f-bbb2-e83153b3c808/'
        'IncomingWebhook/68ac4a1564224e80ad1b3e4d32a88e22/176e0d36-e270-40bb-8b1e-4bfba2c98d82')
    title = 'Check Build Report'
    if good:
        pass
    else:
        myteamsmessage.color('red')
    myteamsmessage.title(title)
    myteamsmessage.text(msg)
    myteamsmessage.send()


def get_white_list():
    with open('whitelist_for_buildcheck', 'r') as f:
        white_jobs = f.readlines()
    new_white_jobs = []
    for it in white_jobs:
        new_it = it.split('\n')[0]
        new_white_jobs.append(new_it)
    return new_white_jobs


def get_timeout_fail():
    # init a timer list and get job_timer_dict
    timer_list = init_timer_list(len(record_unfinished))
    print 'fail_dict_list-INININ-: ', fail_dict_list
    for job_id in xrange(len(record_unfinished)):
        job = record_unfinished[job_id]
        if job in unfinished:
            timer_list[job_id] = 1
            job_timer_dict[job] = timer_list[job_id]
    print 'init job_timer_dict: ', job_timer_dict

    print 'old record_timer_dict: ', record_timer_dict
    for key in record_timer_dict.keys():
        if key in job_timer_dict.keys():
            job_timer_dict[key] = record_timer_dict[key] + job_timer_dict[key]
    print 'after merge from record_timer_dict, job_timer_dict: ', job_timer_dict

    for key in job_timer_dict.keys():
        job_name = key
        # job_name = job_timer_dict.keys()[job_timer_dict.values().index(job_timer_dict[key])]
        if job_timer_dict[key] > 720:  # 720*20sec/3600=4 hours
            print "Task: \'%s\' did not finished in 4 hours, " \
                  "send it in failed list" % job_name
            reason = "\'%s\' did not finished in 4 hours, " \
                     "send it in failed list" % job_name   # send notification-------------->>>>>
            fail.append(job_name)
            print 'fail-----', fail
            print 'fail_dict_list-ououout-: ', fail_dict_list
            fail_dict['fail_reason'] = reason
            fail_dict['Branch'] = job_name
            fail_dict_list.append(fail_dict)
            print 'fail_dict_list--: ', fail_dict_list
        else:
            print "Task: \'%s\' is unfinished, spent time: %s seconds" % \
                  (job_name, job_timer_dict[key] * 20)


def get_cm_start_finish_time(branch, build):
    url_list, page_url, image_url = [], [], []
    start_t, finish_t, msg = '', '', ''

    project_show_url = 'https://builds.wgti.net/project/show/%s' % branch
    body = requests.get(project_show_url, verify=False).text
    soup = BeautifulSoup(body, 'lxml')
    soup.prettify()

    for anchor in soup.findAll('a', href=True):
        url_list.append(anchor.get('href'))
    for url in url_list:
        if re.match('http://cm02se.wgti.net/.*', url):
            page_url.append(url)
        if re.match(r'ftp://cmimages.wgti.net/images/%s/.*' % branch, url):
            image_url.append(url)
    for idx in xrange(len(image_url)):
        if image_url[idx].split('/')[-1] == build:
            wanna_page = page_url[idx]
            body = requests.get(wanna_page, verify=False).text
            print body
            started = re.findall('Build started : \d+-\d+-\d+ \d+:\d+.*', body)
            finished = re.findall('Build finished: \d+-\d+-\d+ \d+:\d+.*', body)
            if not started or not finished:
                started = re.findall('start time.*\d+:\d+:\d+.*', body)
                finished = re.findall('end time.*\d+:\d+:\d+.*', body)
            print 'started, finished: ', started, finished
            start_t = re.findall('\d+.*', started[0])[0]
            finish_t = re.findall('\d+.*', finished[0])[0]
            break
    if not start_t and not finish_t:
        msg = 'the image: %s do not exist in image list of branch: %s, ' \
              'please check on web: %s' % (build, branch, project_show_url)

    return start_t, finish_t, msg

if __name__ == '__main__':
    # hipchat room information
    HIPCHAT_SERVER = 'https://watchguard.hipchat.com'
    HIPCHAT_TOKEN = 'psw9IfRRhVzaHmaJ80JUpQDoJN9h6WtgtSSO8YWx'
    HIPCHAT_ROOM = '3280364'
    # parameters initial
    unfinished, success, fail = [], [], []  # store result: unfinished, success, fail
    record_unfinished = []  # record unfinished
    record_timer_dict = {}  # record total timer of unfinished
    job_timer_dict = {}  # store current timer of unfinished
    success_dict, fail_dict = {}, {}  # for send_notification
    success_dict_list, fail_dict_list = [], []  # for send_notification


    # get white list
    white_list = get_white_list()

    # get all old rows
    old_rows = _get_rows()

    #--------local debug------------
    testfile = r'D:\Project_TCS\git_projects\TCS\farmacias\test_script\dashboard\testfile.txt'
    with open(testfile, 'wb') as f:
        for row in old_rows:
            f.write(row + '\n')
    #exit(0)
    #--------local debug------------

    # get old latest date
    date_list = _get_date(old_rows)
    old_latest_build_date = date_list[0]
    print 'old_latest_build_date: ', old_latest_build_date

    # get old latest row with no date
    row_list = _get_arr_no_date(old_rows)
    old_row1 = row_list[0]
    print 'old_row1: ', old_row1

    # get old status list
    old_status_list = _get_status(old_rows)

    while True:
        # check interval
        time.sleep(20)

        # request website and get new rows of table
        #new_rows = _get_rows()

        #--------local debug------------
        a = []
        with open(testfile, 'r') as f:
            for line in f.readlines():
                #print line
                a.append(line)
        new_rows = []
        for item in a:
            new_rows.append(item.split('\n')[0])
        print new_rows
        # -------local debug-------------

        # get new latest row with no date
        arr_list = _get_arr_no_date(new_rows)
        new_row1 = row_list[0]
        print 'new_row1: ', new_row1

        date_list = _get_date(new_rows)
        status_list = _get_status(new_rows)
        project_list = _get_project(new_rows)

        # get new_latest_build_date
        new_latest_build_date = date_list[0]

        # key point
        try:
            idx = date_list.index(old_latest_build_date.strip())
        except:
            idx = None
        print 'idx: ', idx

        # judge idx
        if idx is None:
            if check_status(status_list[0]):
                success.append(arr_list[0])
            print 'success: ', success
        elif idx == 0:
            unfinished, fail = get_unfinished_and_failed(old_status_list, status_list, project_list)
            print 'unfinished, fail: ', unfinished, fail
        else:
            unfinished, success, fail = get_unfinished_success_fail(idx, status_list, arr_list, record_unfinished)
            print 'unfinished, success, fail: ', unfinished, success, fail

        if success:
            for suc_job in success:
                bran = suc_job.split()[0]
                buil = suc_job.split()[2]
                success_dict['branch'] = bran
                success_dict['build'] = buil
                start_time, finish_time, error = get_cm_start_finish_time(bran, buil)
                if start_time and finish_time:
                    success_dict['start_time'] = start_time
                    success_dict['finish_time'] = finish_time
                else:
                    success_dict['error_info'] = error
                success_dict_list.append(success_dict)
            print 'success_dict_list: ', success_dict_list

            # send job request and notification -- success
            #send_job_request(success)
            send_success_notification()

        # update old latest row and date
        old_row1 = new_row1
        old_latest_build_date = new_latest_build_date

        # add new to record_unfinished and calculator timer
        for job_un in unfinished:
            if job_un not in record_unfinished:
                record_unfinished.append(job_un)

        print 'fail_dict_list: ', fail_dict_list
        # calculate timer and put timeout job into fail list
        get_timeout_fail()
        print 'fail_dict_list: ', fail_dict_list
        # send job request and notification -- fail
        if fail:
            #send_job_request(fail)
            send_fail_notification()

        # update job_timer_dict to record_timer_dict
        # record_timer_dict = job_timer_dict
        record_timer_dict = {'utm-inc': 720}
        print 'record_timer_dict: job_timer_dict', record_timer_dict, job_timer_dict

        print 'unfinished: ', unfinished
        print 'success: ', success
        print 'fail: ', fail

        # update record_unfinished
        record_unfinished = unfinished
        # update old_status_list
        old_status_list = status_list

        # flush parameters
        unfinished, success, fail = [], [], []  # store result: unfinished, success, fail
        job_timer_dict = {}
        success_dict, fail_dict = {}, {}  # for send_notification
        success_dict_list, fail_dict_list = [], []  # for send_notification
        print '-------------------------------------------'

 

转载于:https://www.cnblogs.com/azure-west/p/7575828.html