python脚本（Phabricator）

sssssssssnake

已于 2024-07-16 10:42:39 修改

阅读量211

点赞数 2

分类专栏：小工具文章标签： python 开发语言

于 2024-07-12 17:23:36 首次发布

本文链接：https://blog.csdn.net/sssssssssnake/article/details/140383886

版权

小工具专栏收录该内容

13 篇文章 0 订阅

订阅专栏

ph_ac.py
Phabricator自动accept

import json

import requests

api_token=''

def get_phid(revision_id):
    url = "https://test.com/api/differential.revision.search"

    data = {
        "api.token":api_token,
        "constraints[ids][0]": revision_id,
    }

    response = requests.post(url, data=data)
    response.raise_for_status()
    res = json.loads(response.text)
    return res["result"]["data"][0]["phid"]

def ph_ac(phid):
    url = "https://test.com/api/differential.revision.edit"

    data = {
        "api.token": api_token,
        "transactions[0][type]": "accept",
        "transactions[0][value]": "1",
        "objectIdentifier": phid,
    }

    response = requests.post(url, data=data)
    response.raise_for_status()

    print(response.text)

def main():
    phid=get_phid(426693)
    ph_ac(phid)


if __name__ == '__main__':
    main()

ph_build_callback.py
Phabricator修改build状态

import re
import requests

ph_host=''
api_token=''
phsid = ''

def callback(revision_id, build_name):
    cookies = {
        "phsid": phsid,
    }
    revision_url = f'{ph_host}/D{revision_id}'
    revision_response = requests.get(revision_url, cookies=cookies)
    print(revision_response.text)
    build = re.search(rf'href="(/harbormaster/build/\d+/).*? {build_name}',revision_response.text)
    print(build)
    if build:
        build = build.group(1)
        build_url = f'{ph_host}{build}'
        build_response = requests.get(build_url, cookies=cookies)
        target = re.search(r'target.phid</td><td class="wide">(.*?)</td>',build_response.text)
    print(target)
    if target:
        target_id = target.group(1)
        send_msg_url = f'{ph_host}/api/harbormaster.sendmessage'
        data = {
        'api.token': api_token,
        'type': 'pass',
        'buildTargetPHID': target_id
        }
        message_response = requests.post(send_msg_url,data=data)
        print(message_response.text)

callback(376941,1651469)

query_user_cr.py
查询用户CR

import requests

api_token=''


def get_user_phid(user_name):
    url = "https://test.com/api/user.query"
    headers = {
        "Content-Type": "application/x-www-form-urlencoded; charset=utf-8"
    }
    data = {
        "api.token": api_token,
        "usernames[0]": user_name
    }

    response = requests.post(url, headers=headers, data=data)
    return response.json()['result'][0]['phid']

#<"status-any", "status-open", "status-accepted", "status-needs-review", "status-needs-revision", "status-closed", "status-abandoned">
#<"order-modified", "order-created">
def get_cr(phid,status="status-needs-review",order="order-modified"):
    url = "https://test.com/api/differential.query"
    headers = {
        "Content-Type": "application/x-www-form-urlencoded; charset=utf-8"
    }
    data = {
        "api.token": api_token,
        "authors[0]": phid,
        "status": status,
        "order": order,
        "limit": 10
    }
    response = requests.post(url, headers=headers, data=data)
    for cr in response.json()['result']:
        print(cr)


def main():
    user_phid=get_user_phid("")
    get_cr(user_phid)

if __name__ == '__main__':
    main()

create_ph_repo.py
自动的创建ph的仓库，并配置导入gitlab仓库数据。

import json
import re
import requests
from bs4 import BeautifulSoup


api_token = ""
api_url = "https://test.com"
repository_url = 'https://test.com/diffusion/{}/manage/uris/'
phsid = ''
credential = ""
gitlab_url = "https://gitlab"
gitlab_prefix = "git@gitlab"
input_file = r'input.txt'
repo_file=r'repo.txt'

def repository_activate(repository_id):
    data = {
        "api.token": api_token,
        "transactions[0][type]": "status",
        "transactions[0][value]": "active",
        "objectIdentifier": repository_id,
    }

    response = requests.post(
        api_url + '/api/diffusion.repository.edit',
        data=data)
    response.raise_for_status()
    # print(response.text)

def find_repo_with_ssh_url(repo_url,repo_id=None):
    url = f'{api_url}/api/repository.query'
    limit = 100
    data = {
        'api.token': api_token,
        'limit': limit
    }
    if repo_id:
        data['after'] = repo_id
    response = requests.post(url, data=data).json()
    for result in response['result']:
        repo_id = result['id']
        remote_url = result['name']
        if remote_url==repo_url:
            return True

    if repo_id and len(response['result']) >= limit:
        return find_repo_with_ssh_url(repo_url,repo_id)

    return False


def repository_create(name):
    data = {
        "api.token": api_token,
        "transactions[0][type]": "name",
        "transactions[0][value]": name,
        "transactions[1][type]": "vcs",
        "transactions[1][value]": "git"
    }

    response = requests.post(
        api_url + '/api/diffusion.repository.edit',
        data=data)
    response.raise_for_status()

    res = json.loads(response.text)

    repository_id = res['result']['object']['id']
    repository_phid = res['result']['object']['phid']

    # print(repository_id)
    # print(repository_phid)

    # print(response.text)
    return repository_id, repository_phid


def url_close(id):
    data = {
        "api.token": api_token,
        "transactions[0][type]": "io",
        "transactions[0][value]": "none",
        "transactions[1][type]": "display",
        "transactions[1][value]": "never",
        "objectIdentifier": id
    }

    response = requests.post(
        api_url + '/api/diffusion.uri.edit',
        data=data)
    response.raise_for_status()
    # print(response.text)


def get_url_ids(repository_id):
    cookies = {
        "phsid": phsid,
    }

    response = requests.get(repository_url.format(repository_id),
                            cookies=cookies)
    response.raise_for_status()

    url_ids = []
    soup = BeautifulSoup(response.text, 'html.parser')
    div = soup.find('div', class_='aphront-table-wrap')
    a_tags = div.find_all('a')
    for tag in a_tags:
        url_ids.append(tag['href'].split('/')[-2])

    return url_ids



def url_create(uri, repository_phid):
    data = {
        "api.token": api_token,
        "transactions[0][type]": "repository",
        "transactions[0][value]": repository_phid,
        "transactions[1][type]": "uri",
        "transactions[1][value]": uri,
        "transactions[2][type]": "credential",
        "transactions[2][value]": credential,
        "transactions[3][type]": "io",
        "transactions[3][value]": "observe",
        "transactions[4][type]": "display",
        "transactions[4][value]": "always"
    }

    response = requests.post(
        api_url + '/api/diffusion.uri.edit',
        data=data)
    response.raise_for_status()
    # print(response.text)


def convert_http_to_git(url):
    url=url.strip()
    if url.endswith('/'):
        url=url[0:-1]
    if ".git" not in url:
        url+='.git'

    pattern = gitlab_url + '/(.*)/(.*).git'
    match = re.search(pattern, url)
    if match:
        return gitlab_prefix + f':{match.group(1)}/{match.group(2)}.git'
    else:
        return url


def extract_project_path(git_url):
    path = git_url.split(':')[-1].split('.git')[0]

    return path


def main():
    with open(repo_file, 'r') as repo_input_file:
        repo_set = set()
        lines = repo_input_file.readlines()
        for line in lines:
            repo_set.add(line.strip())

    with open(input_file, 'r') as file:
        create_set = set()
        for line in file:  # 不需要readlines()，直接迭代文件对象即可
            try:
                git_url = convert_http_to_git(line)
                name = extract_project_path(git_url)
                if name in create_set or name in repo_set or name.startswith('3rd') or name.startswith('third_party') or find_repo_with_ssh_url(name):
                    print("仓库已添加")
                    continue
                else:
                    create_set.add(name)
                    repository_id, repository_phid = repository_create(name)
                    url_ids = get_url_ids(repository_id)
                    for id in url_ids:
                        url_close(id)
                    url_create(git_url, repository_phid)
                    repository_activate(repository_id)
                    print(name)
                    print("success")

                    # 将name写入repo_file
                    with open(repo_file,'a') as repo_output_file:  # 使用'a'模式打开文件以追加内容
                        repo_output_file.write(name + '\n')  # 写入name，并添加换行符
            except Exception as e:
                print(e)
                print("fail")
                print(line)


if __name__ == "__main__":
    main()

query_repo.py
查询仓库并写入文件input.txt给上面脚本使用
调用
填入id.txt
python3 query_repo.py
python3 create_ph_repo.py
自动化
0 */1 * * * python3 query_repo.py
30 */1 * * * python3 create_ph_repo.py >> /tmp/create_ph_repo.log 2>&1

import gitlab

# 从配置文件中加载 GitLab 实例
gl = gitlab.Gitlab.from_config('trigger', ['python-gitlab.cfg'])

gl.auth()

# 指定文件名
id_file = 'id.txt'
input_file = 'input.txt'

# 打开文件并读取第一行
with open(id_file, 'r') as file:
    first_line = file.readline()

with open(input_file, 'w') as file:
    pass  # 不需要写入任何内容，只是打开文件即可清空它

# 去除可能存在的空白字符（如空格、换行符等）
first_line = first_line.strip()

# 尝试将第一行转换为整数
try:
    start = int(first_line)
    print("读取到的数字是:", start)
except ValueError:
    print("第一行不是一个有效的数字")

while True:
    retry_count = 0  # 初始化重试计数器
    while retry_count < 5:  # 设置重试次数上限为5
        try:
            # 尝试获取项目
            project = gl.projects.get(start + 1)
            start += 1  # 成功后增加start
            print(f"成功获取项目信息，项目名称：{project.name}，项目ID：{project.id}")
            with open(input_file, 'a') as file:
                # 写入数字字符串到文件
                file.write(project.web_url + "\n")
            with open(id_file, 'w') as file:
                # 写入数字字符串到文件
                file.write(str(start))
            break  # 成功后跳出重试循环
        except gitlab.exceptions.GitlabGetError as e:
            print(f"获取项目信息失败，错误信息：{e}")
            retry_count += 1  # 重试计数器加一
            start+=1
            if retry_count < 5:
                print(f"正在进行第 {retry_count} 次重试...")
            else:
                print("已达到最大重试次数，停止尝试。")
                break  # 达到最大重试次数后跳出整个循环
        except Exception as e:
            print(f"发生未知错误：{e}")
            break  # 发生其他未知错误时，跳出整个循环

    if retry_count >= 5:
        break  # 如果重试次数达到上限或有其他异常，则退出主循环

get_ph_repo_cr_info.py
获取指定仓库的CR提交信息

import datetime
import re
import time
import requests

ph_url = 'https://cr.hobot.cc'
ph_token = ''
# revision转换
revision_search = \
    f'{ph_url}/api/differential.revision.search'
# 用户查询
user_query = f'{ph_url}/api/user.query'
# 分支查询
branch_search = f'{ph_url}/api/differential.diff.search'
repository_search = \
    f'{ph_url}/api/diffusion.repository.search'




def get_ph_branch(revision_phid):
    data = {
        "api.token": ph_token,
        "constraints[revisionPHIDs][0]": revision_phid
    }
    response = requests.post(
        ph_url + '/api/differential.diff.search', data)

    for branch in response.json()['result']['data']:
        branch_name = None
        for ref in branch['fields']['refs']:
            if ref['type'] == 'branch':
                branch_name = ref['name']
            if ref['type'] == 'onto':
                branch_name = ref['name']
        if not branch_name:
            continue
        else:
            return branch_name
    return ''

def get_last_month_day_time(n=7):
    # 今天日期
    today = datetime.date.today()
    # 昨天时间
    last_month_day = today - datetime.timedelta(days=n)
    start = int(
        time.mktime(time.strptime(str(last_month_day), '%Y-%m-%d')))
    if start < 1677945600:
        start = 1677945600
    return start

def get_ph_repo_path(ph_id):
    data = {
        'phids[0]': ph_id,
        'api.token': ph_token
    }
    req = requests.get(
        url=ph_url + '/api/repository.query',
        data=data).json()
    name = req['result'][0]['name']
    if name:
        return name
    else:
        return ''

def get_timestamp(date):
    start = int(
        time.mktime(time.strptime(date, '%Y-%m-%d')))
    return start

def search_revision(repo_phid, after_id=None,
                    last_create_timestamp=get_last_month_day_time(7),ph_info={}):
    data = {
        "api.token": ph_token,
        "constraints[repositoryPHIDs][0]": repo_phid,
        "constraints[createdStart]": last_create_timestamp,
        "limit": 100
    }
    if after_id:
        data['after'] = after_id
    result = requests.get(url=revision_search, params=data)
    result = result.json()['result']

    for res in result['data']:
        after_id = res['id']
        parse_ph_info(res,ph_info)

    if len(result['data']) < int(result['cursor']['limit']):
        return ph_info

    return search_revision(repo_phid, after_id,
                         last_create_timestamp,ph_info)

def search_revision_by_date(repo_phid, after_id=None,
                    last_create_timestamp=0,end_create_timestamp=1711036800,ph_info={}):
    data = {
        "api.token": ph_token,
        "constraints[repositoryPHIDs][0]": repo_phid,
        "constraints[createdStart]": last_create_timestamp,
        "constraints[createdEnd]": end_create_timestamp,
        "limit": 100
    }
    if after_id:
        data['after'] = after_id
    result = requests.get(url=revision_search, params=data)
    result = result.json()['result']

    for res in result['data']:
        after_id = res['id']
        parse_ph_info(res,ph_info)

    if len(result['data']) < int(result['cursor']['limit']):
        return ph_info

    return search_revision_by_date(repo_phid, after_id,
                         last_create_timestamp,end_create_timestamp,ph_info)
def get_lint_passed(buildable_id):
    data = {
        'constraints[buildables][0]': buildable_id,
        'api.token': ph_token
    }
    req = requests.get(
        url=ph_url + '/api/harbormaster.build.search',
        data=data).json()
    ret = req['result']['data']
    for i in ret:
        if i['fields']['name'] == "CI_SCM_Force_LINT_Check":
            if i['fields']['buildStatus']['value'] == 'passed':
                return True
    return False

def get_username(phid):
    data = dict()
    data['api.token'] = ph_token
    data['phids[0]'] = phid
    response = requests.post(user_query, data=data).json()
    if response['result']:
        return response['result'][0]['userName']

def get_db_repo(git_url):

    git_path = re.search(r'gitlab.hobot.cc[:/](.*?)\.git', git_url)
    if git_path:
        ph_name = git_path.group(1)
        # print(ph_name)
    data = {"api.token": ph_token,
            'constraints[query]': ph_name}
    result = requests.get(url=repository_search, params=data)
    for res in result.json()['result']['data']:
        if res['fields']['name'].strip() == ph_name.strip():
            # print(res['phid'])
            return res['phid']

    return ''


def parse_ph_info(data,ph_info):
    revision_id = data['id']
    if revision_id not in ph_info.keys():
        ph_info[revision_id] = dict()
    #print(data['fields']['uri'])

    ph_info[revision_id]['status']=data['fields']['status']['name']
    ph_info[revision_id]['title'] = data['fields']['title'].strip()
    ph_info[revision_id]['uri'] = data['fields']['uri']
    ph_info[revision_id]['timeSpent'] = data['fields']['timeSpent']
    ph_info[revision_id]['dateCreated'] = data['fields']['dateCreated']
    ph_info[revision_id]['dateModified'] = data['fields']['dateModified']
    ph_info[revision_id]['author'] = get_username(data['fields']['authorPHID'])

    data = dict()
    data['api.token'] = ph_token
    data['ids[0]'] = revision_id
    differential_query = requests.post(
        ph_url + '/api/differential.query', data=data).json()
    #print(differential_query)

    # ph_info[revision_id]['lint_passed'] = False

    # buildable_id = list(differential_query['result'][0]['properties'][
    #                         'buildables'].keys())[-1]
    # build_status = differential_query['result'][0]['properties'][
    #     'buildables'][buildable_id]['status']
    # ph_info[revision_id]['build_status'] = build_status
    # if get_lint_passed(buildable_id):
    #     ph_info[revision_id]['lint_passed'] = True

    ph_info[revision_id]['closed'] = False
    if differential_query['result'][0]['statusName'] == 'Closed':
        ph_info[revision_id]['closed'] = True
    if differential_query['result'][0]['statusName'] in ['Accepted',
                                                         'Closed']:
        ph_info[revision_id]['ac'] = True
    else:
        ph_info[revision_id]['ac'] = False

    # if differential_query['result'][0]['repositoryPHID']:
    #     ph_info[revision_id]['repo'] = True
    #
    #     repo_path = get_ph_repo_path(
    #         differential_query['result'][0]['repositoryPHID'])
    #     ph_info[revision_id]['repo_path'] = repo_path

    # else:
    #     ph_info[revision_id]['repo'] = False

    # ph_info[revision_id]['testPlan'] = differential_query['result'][0]['testPlan']
    #
    # ph_info[revision_id]['commits'] = list()
    # if ph_info[revision_id]['ac']:
    # for commit_data in differential_query['result'][0]['hashes']:
    #     if commit_data[0] == 'gtcm':
    #         ph_info[revision_id]['commits'].append(commit_data[1])
    #

    ph_info[revision_id]['reviewers'] = list()
    for phid in differential_query['result'][0]['reviewers']:
        if not phid.find('PHID-USER-') == 0:
            continue
        ph_info[revision_id]['reviewers'].append(get_username(phid))

    diff_id = differential_query['result'][0]['diffs'][0]
    ph_id = differential_query['result'][0]['phid']
    ph_info[revision_id]['branch'] = get_ph_branch(ph_id)
    # ph_info[revision_id]['phid'] = ph_id
    ph_info[revision_id]['diff_id'] = diff_id
    #
    data = dict()
    data['api.token'] = ph_token
    data['ids[0]'] = diff_id
    differential_querydiffs = requests.post(
        ph_url + '/api/differential.querydiffs', data=data).json()

    changes = differential_querydiffs['result'][diff_id]['changes']
    ph_info[revision_id]['file_count'] = 0
    ph_info[revision_id]['line_count'] = differential_query['result'][0]['lineCount']
    ph_info[revision_id]['addLines'] = 0
    ph_info[revision_id]['delLines'] = 0
    ph_info[revision_id]['path_change'] = dict()
    # print('the changes is:' + str(changes))
    for change in changes:
        ph_info[revision_id]['file_count'] = \
            ph_info[revision_id]['file_count'] + 1
        addLines = int(change['addLines'])
        delLines = int(change['delLines'])
        ph_info[revision_id]['addLines'] += addLines
        ph_info[revision_id]['delLines'] += delLines
    ph_info[revision_id]['accepter'] = list()
    ph_info[revision_id]['request_changer'] = list()
    ph_info[revision_id]['comments'] = dict()
    #
    data = dict()
    data['api.token'] = ph_token
    data['objectIdentifier'] = ph_id
    data['limit'] = 99999
    transaction_search = requests.post(
        ph_url + '/api/transaction.search', data).json()
    #print(transaction_search['result']['data'])
    for i in transaction_search['result']['data']:
        if i['authorPHID'] == 'PHID-APPS-PhabricatorHarbormasterApplication':
            continue
        if i['type'] == 'accept':
            user = get_username(i['authorPHID'])
            if user:
                ph_info[revision_id]['accepter'].append(user)
        if i['type'] == 'request-changes':
            user = get_username(i['authorPHID'])
            ph_info[revision_id]['request_changer'].append(user)
        if not i['type'] == 'comment' and not i['type'] == 'inline':
            continue
        if not i['authorPHID'].find('PHID-USER-') == 0:
            continue
        if i['comments']:
            if i['comments'][0]['removed']:
                continue
            username = get_username(i['authorPHID'])
            if username == 'robot':
                continue
            ph_info[revision_id]['comments'][username]=ph_info[revision_id]['comments'].get(username,0)+1



# repo_phid=get_db_repo("git@gitlab.hobot.cc:auto/holm/holm/navinet_localization.git")
# ph_info=search_revision(repo_phid,None,get_last_month_day_time(7))
# print(len(ph_info.keys()))

def get_ph_cr_info(request):
    # 从请求中获取参数
    git_url = request.GET.get('git_url')  # 如果是GET请求
    # 或者
    # repo_url = request.POST.get('repo_url')  # 如果是POST请求
    before_day = request.GET.get('before_day')  # 同理，如果是GET请求
    # 或者
    # before_day = request.POST.get('before_day')  # 如果是POST请求

    try:
    # 对参数进行处理，这里只是一个简单的例子
        if git_url:
            git_url = git_url.strip()  # 去除首尾空格
        if before_day:
            before_day = int(before_day)  # 转换为整数，这里假设before_day是一个整数
        repo_phid=get_db_repo(git_url)
        if before_day>90:
            before_day=90
        data=search_revision(repo_phid,None,get_last_month_day_time(before_day),{})
    except Exception as e:
        return JsonResponse({"error":e})
    # 返回JsonResponse
    return JsonResponse({"data":data})

def get_ph_cr_info_by_date(request):
    # 从请求中获取参数
    git_url = request.GET.get('git_url')  # 如果是GET请求
    # 或者
    # repo_url = request.POST.get('repo_url')  # 如果是POST请求
    start_day = request.GET.get('start_day')  # 同理，如果是GET请求
    end_day = request.GET.get('end_day')  # 同理，如果是GET请求
    # 或者
    # before_day = request.POST.get('before_day')  # 如果是POST请求

    try:
    # 对参数进行处理，这里只是一个简单的例子
        if git_url:
            git_url = git_url.strip()  # 去除首尾空格
        repo_phid=get_db_repo(git_url)
        data=search_revision_by_date(repo_phid,None,get_timestamp(start_day),get_timestamp(end_day),{})
    except Exception as e:
        return JsonResponse({"error":e})
    # 返回JsonResponse
    return JsonResponse({"data":data})

PhPolicy.py
ph仓库权限同步

#!/usr/bin/python

import os
import time
import requests
import gitlab
import platform
import pickle
from gitlab_app import models


class PhInfo(object):

    def __init__(self, action=None):
        self.action = action
        self.api_token = ''
        self.domain = 'https://cr.hobot.cc/api/'
        self.query_repo_phid = None
        self.query_project_name = None
        path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        self.view_user = list()
        self.init_view_user(path)
        gitlab_cfg = os.path.join(path, 'python-gitlab.cfg')
        self.gl = gitlab.Gitlab.from_config('trigger', [gitlab_cfg])
        self.project_hash = {
        }
        self.handle_git_repo = [
        ]
        self.name_hash = {
        }
        self.num = 0

    def init_view_user(self, path):
        file_name = f'{path}/ph_view_user.txt'
        with open(file_name, 'r') as f:
            users = f.read()
        self.view_user = [user.strip() for user in users.split('\n')]

    # def get_project_query(self):
    #     url = f'{self.domain}project.query'
    #     data = {
    #         'api.token': self.api_token,
    #         'phids[0]': 'PHID-PROJ-36hhpfuobltnxj426llr',
    #         'phids[1]': 'PHID-PROJ-jpwf3vztmlj5khsrnmda',
    #     }
    #     response = requests.post(url, data=data).content.decode('utf-8')
    #     return response

    # # 查找repo
    # def find_all_repo(self, repo_id=None):
    #     url = f'{self.domain}diffusion.repository.search'
    #     limit = 100
    #     data = {
    #         'api.token': self.api_token,
    #         'limit': limit,
    #         'queryKey': 'active',
    #         'attachments[policy]': '1',
    #     }
    #     if repo_id:
    #         data['after'] = repo_id
    #
    #     response = requests.post(url, data=data).json()
    #     if response['error_code']:
    #         print(response)
    #         return []
    #     print(2, response)
    #     repo_id = None
    #     for result in response['result']['data']:
    #         repo_id = result['id']
    #
    #     return_length = len(response['result'])
    #     if repo_id and return_length >= limit:
    #         self.find_all_repo(repo_id)

    # # 查找repo
    # def find_one_repo(self, repo_name='iot/experimental/liyu-test'):
    #     url = f'{self.domain}diffusion.repository.search'
    #     data = {
    #         'api.token': self.api_token,
    #         'constraints[query]': repo_name,
    #         # 'queryKey': 'active',
    #         'attachments[policy]': '1',
    #     }
    #
    #     response = requests.post(url, data=data)
    #     print(3, response.json())
    #     return response

    # 查找repo
    def find_repo_with_ssh_url(self, repo_id=None, git_addr=None,
                               bpm_member_list=None):
        if bpm_member_list and isinstance(bpm_member_list, str):
            bpm_member_list = [bpm_member_list]
        url = f'{self.domain}repository.query'
        limit = 100
        data = {
            'api.token': self.api_token,
            'limit': limit
        }
        if repo_id:
            data['after'] = repo_id
        response = requests.post(url, data=data).json()
        for result in response['result']:
            repo_id = result['id']
            remote_url = result['name']
            is_active = result['isActive']
            if not is_active:
                continue
            # 不是 git@gitlab.hobot.cc 开头不查询
            # if 'gitlab.hobot.cc' not in remote_url:
            #     continue
            #
            # git_dir = re.search(r'gitlab\.hobot\.cc[:/](.*?)(\.git|)$',
            #                     remote_url)
            # if git_dir:
            #     remote_url = git_dir.group(1)
            # else:
            #     continue
            try:
                # 有git地址走这里
                if git_addr:
                    if isinstance(git_addr, list):
                        if remote_url in git_addr:
                            self.handle_ph(result, remote_url, bpm_member_list)
                    else:
                        if remote_url == git_addr:
                            self.handle_ph(result, remote_url, bpm_member_list)
                            return

                # 全部查询走这里
                else:
                    self.handle_ph(result, remote_url, bpm_member_list)
            except Exception as err:
                print(f'{remote_url} err :{err}')

        if repo_id and len(response['result']) >= limit:
            self.find_repo_with_ssh_url(repo_id, git_addr, bpm_member_list)
        print(self.num)

    def ph_repo_can_edit(self, repo_id):
        repo = self.gl.projects.get(repo_id).attributes
        if repo['visibility'] == 'private':
            return True

        return False

    def handle_ph(self, result, remote_url, bpm_member_list):
        repo_phid = result['phid']
        project_name = result['name']
        if self.action == 'update':
            git_repo_info = self.get_db_repo_with_addr(remote_url)
            if git_repo_info is None:
                git_repo_info = self.search_git_repo(remote_url)
            git_repo_id = git_repo_info['id']
            if not self.ph_repo_can_edit(git_repo_id):
                return
            self.num += 1
            if bpm_member_list:
                can_del = False
                ph_member = self.get_phid_user(bpm_member_list)
            else:
                git_member = self.get_git_info_by_id(
                    git_repo_info.get('id'))
                ph_member = self.get_phid_user(git_member)
                can_del = True
            if isinstance(ph_member, bool):
                print(f'{remote_url} 获取成员失败， 该仓库暂停同步')
                return
            elif isinstance(ph_member, list) and len(ph_member) > 0:
                project_id, add_ph_user_ids, del_project_user = \
                    self.find_project(project_name, ph_member)
                if project_id is False:
                    print(f'{remote_url} 获取project_id失败， 该仓库暂停同步')
                    return
                edit_res = self.project_edit(project_id, add_ph_user_ids,
                                             del_project_user, can_del)
                if edit_res is False:
                    print(f'{remote_url} 编辑project_id失败， 该仓库暂停同步')
                    return
                policy_res = self.set_repo_policy(repo_phid, project_id)
                if policy_res is False:
                    print(f'{remote_url} 编辑policy失败， 该仓库暂停同步')
                    return
            else:
                print(f'{remote_url} 获取成员为空， 该仓库暂停同步')
                return
        elif self.action == 'rollback':
            self.set_repo_policy(repo_phid, 'users')

    # repo policy
    def set_repo_policy(self, repo_ph_id, project_ph_id):
        url = f'{self.domain}diffusion.repository.edit'
        data = {
            'api.token': self.api_token,
            'transactions[view][type]': 'view',
            'transactions[view][value]': project_ph_id,
            'transactions[edit][type]': 'edit',
            'transactions[edit][value]': project_ph_id,
            # 'transactions[policy.push][type]': 'policy.push',
            # 'transactions[policy.push][value]': project_ph_id,
            'objectIdentifier': repo_ph_id,
        }
        response = requests.post(url, data=data).json()
        if response['error_code']:
            return False
        return True

    @staticmethod
    def get_db_repo_with_addr(git_addr):
        ssh_url = f'git@gitlab.hobot.cc:{git_addr}.git'
        repo = models.Repo.objects.filter(
            ssh_url_to_repo=ssh_url)
        if repo:
            return {
                'id': repo[0].repo_id
            }
        return None

    @staticmethod
    def get_db_repo_with_repo_id(repo_id):
        repo = models.Repo.objects.filter(
            repo_id=repo_id)
        if repo:
            return {
                'id': repo[0].repo_id
            }
        return None

    # --------------------------------------------------------------
    # 查询ph user
    def get_phid_user(self, user_list):
        url = f'{self.domain}user.query'
        step = 80
        data_list = [user_list[i:i + step] for i in range(
            0, len(user_list), step)]
        return_users = []
        for d in data_list:
            data = {
                'api.token': self.api_token,
            }
            for index, username in enumerate(d):
                if self.name_hash.get(username, None):
                    username = self.name_hash.get(username)
                data[f'usernames[{index}]'] = username
            response = requests.post(url, data=data).json()
            if response['error_code']:
                return False
            for user in response['result']:
                return_users.append(user['phid'])
        return return_users

    @staticmethod
    def get_db_ph_user(user_list):
        email_list = [user + '@horizon.ai' for user in user_list]
        users = models.People.objects.filter(email__in=email_list)
        db_user = list()
        user_set = set()
        for user in users:
            if not user.ph_id:
                continue
            username = user.email.replace('@horizon.ai', '')
            db_user.append(user.ph_id)
            user_set.add(username)
        search_user = set(user_list) - user_set
        return db_user, list(search_user)

    @staticmethod
    def update_db_ph_user(user_list):
        email_list = [user + '@horizon.ai' for user in user_list]
        users = models.People.objects.filter(email__in=email_list)
        return users

    # 查询project
    def find_project(self, project_name, gitlab_member):
        if self.project_hash.get(project_name, None):
            project_name = self.project_hash.get(project_name)
        # 查找project
        url = f'{self.domain}project.search'
        data = {
            'api.token': self.api_token,
            'queryKey': "all",
            'constraints[name]': project_name,
            'attachments[members]': True,
        }
        response = requests.post(url, data=data).json()
        data = response['result']['data']
        project_ph_id = None
        project_user_ph_id = list()
        for project in data:
            if project['fields']['name'] == project_name:
                project_ph_id = project['phid']
                members = project['attachments']['members']['members']
                for member in members:
                    project_user_ph_id.append(member['phid'])
                break
        if project_ph_id is None:
            # 创建 group
            url = f'{self.domain}project.create'
            data = {
                'api.token': self.api_token,
                'name': project_name,
            }
            for index, member in enumerate(gitlab_member):
                data[f'members[{index}]'] = member
            response = requests.post(url, data=data).json()
            if response['error_code']:
                return False, False, False
            project_ph_id = response['result']['phid']
        del_project_user = list(set(project_user_ph_id) - set(gitlab_member))
        add_ph_user_ids = list(set(gitlab_member) - set(project_user_ph_id))
        return project_ph_id, add_ph_user_ids, del_project_user

    # 新增 project member
    def project_edit(self, project_ph_id, add_ph_user_ids,
                     del_project_user, can_del):
        url = f'{self.domain}project.edit'
        data = {
            'api.token': self.api_token,
            'transactions[view][type]': 'view',
            'transactions[view][value]': project_ph_id,
            'transactions[edit][type]': 'edit',
            'transactions[edit][value]': 'admin',
            # 'transactions[join][type]': 'join',
            # 'transactions[join][value]': 'users',
            'objectIdentifier': project_ph_id  # 项目
        }
        if add_ph_user_ids:
            data['transactions[members.add][type]'] = 'members.add'
            for index, username in enumerate(add_ph_user_ids):
                data[f'transactions[members.add][value][{index}]'] = username
        if del_project_user and can_del:
            data['transactions[members.remove][type]'] = 'members.remove'
            for index, username in enumerate(del_project_user):
                data[
                    f'transactions[members.remove][value][{index}]'] = username
        response = requests.post(url, data=data).json()
        if response['error_code']:
            return False
        return True

    # -----------------------------------------------------------
    # policy
    def whoami(self):
        url = f'{self.domain}user.whoami?access_token={self.api_token}'
        data = {
            'api.token': self.api_token
        }
        requests.post(url, data=data)

    # 获取git仓库member
    def get_git_info_by_id(self, repo_id):
        members = self.get_member_from_pkl(repo_id)
        # members = []
        user_list = []
        user_list.extend(self.view_user)

        if members:
            user_list.extend(members)
        else:
            members = self.gl.projects.get(repo_id).members_all.list(all=True)
            for member in members:
                if member.attributes['state'] == 'blocked':
                    continue
                if member.attributes['access_level'] <= 10:
                    continue
                user_list.append(member.attributes['username'])
        return user_list

    def get_member_from_pkl(self, repo_id):
        if platform.system() == 'Windows':
            file = './members_attributes.pkl'
        else:
            file = f'/opt/gitlab_report/raw_data/repo/' \
                   f'{repo_id}/members_attributes.pkl'
        users = self.read_pkl(file)
        if not users:
            return None
        user_list = list()
        for user in users:
            if user['state'] == 'blocked':
                continue
            if user['access_level'] <= 10:
                continue
            user_list.append(user['username'])
        return user_list

    @staticmethod
    def read_pkl(file_name):
        if not os.path.isfile(file_name):
            print(file_name, 'file not found')
            return dict()
        now = time.time()
        fileinfo = os.stat(file_name)
        if now - fileinfo.st_mtime > 3600 * 24:
            return dict()
        f_obj = open(file_name, 'rb')
        return pickle.load(f_obj)

    # 查询 repo id
    def search_git_repo(self, git_addr):
        git_name = git_addr.split('/')[-1]
        name = git_name.replace('.git', '')
        response = self.gl.search('projects', name)
        git_format_addr = f'git@gitlab.hobot.cc:{git_addr}.git'
        for repo in response:
            if repo['ssh_url_to_repo'] == git_format_addr:
                git_repo = self.gl.projects.get(repo.get('id')).attributes
                db_repo = models.Repo.objects.filter(
                    repo_id=repo['id'])
                if db_repo:
                    db_repo[0].ssh_url_to_repo = repo['ssh_url_to_repo']
                    db_repo[0].save()
                else:
                    new_repo = models.Repo(
                        repo_id=git_repo.get('id'),
                        name=git_repo.get('name'),
                        project='',
                        repo_access_level='',
                        apply_date='',
                        create_date=git_repo.get('created_at').split('T')[0],
                        project_last_activity_at=git_repo.get(
                            'last_activity_at', '').split('T')[0],
                        apply_dept='',
                        apply_project_manager='',
                        apply_user='',
                        apply_comments='',
                        repo_subgroup_url='',
                        web_url=git_repo.get('web_url'),
                        project_name='',
                        archived=git_repo.get('archived'),
                        ssh_url_to_repo=git_repo.get('ssh_url_to_repo')
                    )
                    new_repo.save()
                return repo

sssssssssnake

关注

2
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
python脚本（Phabricator）

自动的创建ph的仓库，并配置导入gitlab仓库数据。查询仓库并写入文件input.txt给上面脚本使用。Phabricator修改build状态。Phabricator自动accept。
复制链接

扫一扫

专栏目录