git clone 整个 group

前言

今天跟同事交接项目组的工作，从公司的gitlab上看到，在这个group下面有很多仓库，且各个还有很多按业务分类的文件夹。我想将全部仓库全部clone到本地，但是git clone 不支持groups。用一个图更直观说明我的诉求，YM801这个组织下面有很多分类的文件夹和仓库。

正文

这时候只能google了，好在gitlab提供了api https://docs.gitlab.com/ee/api/groups.html,然后找到了两个重要的很好用的api

1.查询groups的api

GET /groups

返回结果如下

[
  {
    "id": 1,
    "name": "Foobar Group",
    "path": "foo-bar",
    "description": "An interesting group",
    "visibility": "public",
    "share_with_group_lock": false,
    "require_two_factor_authentication": false,
    "two_factor_grace_period": 48,
    "project_creation_level": "developer",
    "auto_devops_enabled": null,
    "subgroup_creation_level": "owner",
    "emails_disabled": null,
    "mentions_disabled": null,
    "lfs_enabled": true,
    "default_branch_protection": 2,
    "avatar_url": "http://localhost:3000/uploads/group/avatar/1/foo.jpg",
    "web_url": "http://localhost:3000/groups/foo-bar",
    "request_access_enabled": false,
    "full_name": "Foobar Group",
    "full_path": "foo-bar",
    "file_template_project_id": 1,
    "parent_id": null,
    "created_at": "2020-01-15T12:36:29.590Z"
  }
]

2.查询对应group下的porject

GET /groups/:id/projects

返回结果如下

[
  {
    "id": 9,
    "description": "foo",
    "default_branch": "master",
    "tag_list": [], //deprecated, use `topics` instead
    "topics": [],
    "archived": false,
    "visibility": "internal",
    "ssh_url_to_repo": "git@gitlab.example.com/html5-boilerplate.git",
    "http_url_to_repo": "http://gitlab.example.com/h5bp/html5-boilerplate.git",
    "web_url": "http://gitlab.example.com/h5bp/html5-boilerplate",
    "name": "Html5 Boilerplate",
    "name_with_namespace": "Experimental / Html5 Boilerplate",
    "path": "html5-boilerplate",
    "path_with_namespace": "h5bp/html5-boilerplate",
    "issues_enabled": true,
    "merge_requests_enabled": true,
    "wiki_enabled": true,
    "jobs_enabled": true,
    "snippets_enabled": true,
    "created_at": "2016-04-05T21:40:50.169Z",
    "last_activity_at": "2016-04-06T16:52:08.432Z",
    "shared_runners_enabled": true,
    "creator_id": 1,
    "namespace": {
      "id": 5,
      "name": "Experimental",
      "path": "h5bp",
      "kind": "group"
    },
    "avatar_url": null,
    "star_count": 1,
    "forks_count": 0,
    "open_issues_count": 3,
    "public_jobs": true,
    "shared_with_groups": [],
    "request_access_enabled": false
  }
]

观察第二个api，有两个很重要的返回字段 ssh_url_to_repo 和 http_url_to_repo ，这不就是我们需要clone的地址吗，皇天不负有心人，有了这两个api就可以完美解决我们的问题了。接下来要准备access_token，这个就不赘述了，关于怎么获取access_token，自行百度。

同时衍生出了第二个问题，我想让clone下来的仓库，跟远程仓库保持同样的文件夹结构，而不是所有的仓库都在同一个文件夹下。万幸的是，咱的电脑上之前装过python，所以我选择用python脚本实现我的需求，这就体现出了脚本语言的好处，不管你精通不精通，看着教程就能写，哈哈哈。

脚本如下：

from urllib.request import urlopen
import json
import subprocess,shlex
import os

#获取当前文件路径
parentpath = os.getcwd()
print("parentpath " + parentpath)
#获取所有group,其中private_token就是access_token
allgroups = urlopen("http://192.168.30.151:80/api/v4/groups?private_token=xQxdakxYNxxxxxxx")
allgroupsDict = json.loads(allgroups.read().decode())
for thisgroup in allgroupsDict:
    try:
	   #切换到父目录
        os.chdir(parentpath)
        thisGroupFullPath = thisgroup['full_path']
        thisGroupId = thisgroup['id']
        print("group full path %s, groupId,%s" % (thisGroupFullPath,thisGroupId))
        #这里我只clone YM801这个group下的仓库
        if "YM801" in thisGroupFullPath:
            isExists = os.path.exists(parentpath+"/"+thisGroupFullPath)
            if not isExists:
                os.makedirs(parentpath+"/"+thisGroupFullPath)
            #保持文件目录与远程仓库一直，并切换到目录
            os.chdir(parentpath+"/"+thisGroupFullPath)
            #获取当前group下的所有仓库
            projects = urlopen("http://192.168.30.151:80/api/v4/groups/"+str(thisGroupId)+"/projects?private_token=xQxdakxYNxxxxxxx")
            projectsDict = json.loads(projects.read().decode())
            for thisProject in projectsDict:
                try:
				  #因为我本地git clone 配置的是http格式的，所以我选择了http_url_to_repo, 如果你是用的git@格式，你就选择ssh_url_to_repo
                    thisProjectURL = thisProject['http_url_to_repo']
                    command = shlex.split('git clone %s' % thisProjectURL)
                    reresultCode = subprocess.Popen(command)
                    print("project url %s" % (thisProjectURL))
        
                except Exception as e:
                    print("Error on %s:%s" % (thisProjectURL,e.strerror))
        
    except Exception as e:
        print("Error on %s:%s" % (thisGroupFullPath,e.strerror))
        

运行以上脚本完美解决我的需求。

总结

以上就是我的处理方案，如有不妥之处，欢迎留言讨论