git仓库迁移（保留提交记录）

迁移项目需要用到一个开源的项目：https://github.com/samrocketman/gitlab-mirrors

同时上传了一个修改后的：git仓库————迁移脚本-Python文档类资源-CSDN下载

这里仅介绍如果把老Git库项目迁移到新Git库，不涉及更新库与删除库

一、原理介绍

主要用到开源项目中的三个文件：config.sh -> add_mirror.sh -> lib/manage_gitlab_project.py，之后会详细讲解这三个文件；

脚本主要是shell+python构成，运行方式比较简单，我这里使用的是Git Bash Here来运行shell命令的，首先进入到gitlab-mirrors目录下，右键即可，如下图：

 运行命令如下：

./add_mirror.sh --git --group-id 组的id --project-name abc --mirror http://账号:访问令牌@192.168.15.20/abc.git

运行原理：

        1. 在运行的电脑上，将老的Git镜像下来

        2. 在新的Git服务器上创建项目

        3. 将运行电脑上的Git remote设置为新的Git地址

        4. 将运行电脑上的Git同步到新的Git地址

二、配置文件介绍（config.sh）

#Environment file

#
# gitlab-mirrors settings
#

#The user git-mirrors will run as.(随便命名)
system_user="gitmirror"
#The home directory path of the $system_user
# 旧Git与新Git的同步中转站，保证此目录为空即可
user_home="d:/gitmirror/"
#The repository directory where gitlab-mirrors will contain copies of mirrored
#repositories before pushing them to gitlab.
# 具体的中转项目，会在此目录下创建对应的项目
repo_dir="${user_home}/repositories"
#colorize output of add_mirror.sh, update_mirror.sh, and git-mirrors.sh
#commands.（不用管）
enable_colors=true
#These are additional options which should be passed to git-svn.  On the command
#line type "git help svn" （不用管）
git_svn_additional_options="-s"
#Force gitlab-mirrors to not create the gitlab remote so a remote URL must be
#provided. (superceded by no_remote_set)（不用管）
no_create_set=false
#Force gitlab-mirrors to only allow local remotes only.（不用管）
no_remote_set=false
#Enable force fetching and pushing.  Will overwrite references if upstream
#forced pushed.  Applies to git projects only.（不用管）
force_update=false
#This option is for pruning mirrors.  If a branch is deleted upstream then that
#change will propagate into your GitLab mirror.  Aplies to git projects only.
prune_mirrors=false

#
# Gitlab settings
#

#This is the base web url of your Gitlab server.
#新Git服务地址
gitlab_url="http://192.168.15.20"
#Special user you created in Gitlab whose only purpose is to update mirror sites
#and admin the $gitlab_namespace group.
#新Git服务器上的账号
gitlab_user=git账号名字
#Generate a token for your $gitlab_user and set it here.
#新Git服务器上生成的访问令牌，后面会讲解如何生成
gitlab_user_token_secret=访问令牌
#Sets the Gitlab API version, either 3 or 4
# 不用管
gitlab_api_version=4
#Verify signed SSL certificates?
# 不用管
ssl_verify=true
#Push to GitLab over http?  Otherwise will push projects via SSH.
# 不用管
http_remote=false

#
# Gitlab new project default settings.  If a project needs to be created by
# gitlab-mirrors then it will assign the following values as defaults.
#

#values must be true or false
# 是否开启讨论功能
issues_enabled=false
wall_enabled=false
# 是否开启Wiki功能
wiki_enabled=true
snippets_enabled=false
#是否开启merge功能
merge_requests_enabled=true
public=false

具体每个字段的含义，在上面的脚本中已经注明

访问令牌的生成，如下图所示：

 

 三、同步脚本（add_mirror.sh）

#!/bin/bash
#Created by Sam Gleske
#MIT License
#Created Tue Sep 10 23:01:08 EDT 2013
#USAGE
#  ./add_mirror.sh --git --project-name someproject --mirror http://example.com/project.git

#bash option stop on first error
set -e

#Include all user options and dependencies
# 添加所有的依赖库
git_mirrors_dir="${0%/*}"
source ${git_mirrors_dir}/includes.sh

#check if api version is set
[ -z $gitlab_api_version ] && gitlab_api_version=4

#export env vars for python script
# 导出config.sh中配置的环境变量，python中也可以用
export gitlab_user_token_secret gitlab_url gitlab_user ssl_verify gitlab_api_version

PROGNAME="${0##*/}"
PROGVERSION="${VERSION}"

#Default script options
git=false
project_name=""
mirror=""
group_id=0 # 从命令行读取的组ID
desc=""
force=false
no_create_set="${no_create_set:-false}"
no_remote_set="${no_remote_set:-false}"
http_remote="${http_remote:-false}"

# 自定义变量
# 通过组ID获取到的组的路径名字
group_name="" # 组对应的路径名字

#
# ARGUMENT HANDLING
#

usage()
{
  cat <<EOF
${PROGNAME} ${PROGVERSION} - MIT License by Sam Gleske

USAGE:
  ${PROGNAME} TYPE --project NAME --mirror URL [--authors-file FILE]

DESCRIPTION:
  This will add a git repository to be mirrored by GitLab.  It
  first checks to see if the project exists in gitlab.  If it does
  not exist then it creates it.  It will then clone and check in the
  first copy into GitLab.  From there you must use the update_mirror.sh
  script or git git-mirrors.sh script.

  -h,--help          Show help

  -v,--version       Show program version

  -f,--force         Force add project even if it already exists.
                     Any program errors will automatically continue.

  -m,--mirror URL    Repository URL to be mirrored.

  -n,--no-create URL Set a static remote without attempting to resolve
                     the remote in GitLab.  This allows more generic
                     mirroring without needing to be specifically for
                     GitLab.

  -l,--no-remote     This is a local only mirror.  There is no remote
                     to push to.  This is meant for mirroring remote
                     projects on a developer machine.

  -g,--group-id     目标git的组Id

  --desc            git的描述

  -p,--project-name NAME
                     Set a GitLab project name to NAME.

REPOSITORY TYPES:
  At least one repository TYPE is required.

  --git              Mirror a git repository (must be explicitly set)

EOF
}
#Short options are one letter.  If an argument follows a short opt then put a colon (:) after it
SHORTOPTS="hvflm:n:p:"
# 如果要在命令行里加参数，需要在下方添加，用于读取命令行中的数据
LONGOPTS="help,version,force,git,mirror:,no-create:,desc:,group-id:,no-remote,project-name:,authors-file:"
ARGS=$(getopt -s bash --options "${SHORTOPTS}" --longoptions "${LONGOPTS}" --name "${PROGNAME}" -- "$@")
eval set -- "$ARGS"
while true; do
  case $1 in
    -h|--help)
        usage
        exit 1
      ;;
    -v|--version)
        echo "${PROGNAME} ${PROGVERSION}"
        exit 1
      ;;
    --git)
        git=true
        shift
      ;;
    -f|--force)
        force=true
        set +e
        shift
      ;;
    -p|--project-name)
        project_name="${2}"
        shift 2
      ;;
    -m|--mirror)
        mirror="${2}"
        shift 2
      ;;
    -n|--no-create)
        no_create_set=true
        no_create="${2}"
        shift 2
      ;;
    -l|--no-remote)
        no_remote_set=true
        shift 1
      ;;
    --group-id)
        group_id="${2}"
        shift 2
      ;;
    --desc)
        desc="${2}"
        shift 2
      ;;
    --authors-file)
        authors_file="${2}"
        shift 2
      ;;
    --)
        shift
        break
      ;;
    *)
        shift
      ;;
    esac
done

#
# Program functions
#

function preflight() {
  STATUS=0
  #test for multiple repository types
  types=0
  selected_types=()
  if ${git};then
    ((types += 1))
    selected_types+=('--git')
  fi
  if [ "${types}" -eq "0" ];then
    red_echo -n "Must select at least one repository type.  e.g. "
    yellow_echo "--git"
    STATUS=1
  elif [ "${types}" -gt "1" ];then
    red_echo -n "Multiple repository types not allowed.  Found:"
    for x in ${selected_types[@]};do
      yellow_echo -n " $x"
    done
    echo ""
    STATUS=1
  fi
  #test required project_name option
  if [ -z "${project_name}" ];then
    red_echo -n "Missing "
    yellow_echo -n "--project-name"
    red_echo " option."
    STATUS=1
  fi
  #test required mirror option
  if [ -z "${mirror}" ];then
    red_echo -n "Missing "
    yellow_echo -n "--mirror"
    red_echo " option."
    STATUS=1
  fi
  #test no_create_set environment variable (must be bool)
  if [ ! "${no_create_set}" = "true" ] && [ ! "${no_create_set}" = "false" ];then
    red_echo -n "no_create_set="
    yellow_echo -n "${no_create_set}"
    red_echo -n " is not a valid option for no_create_set!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  elif ${no_create_set} && [ -z "${no_create}" ];then
    yellow_echo -n "--no-create"
    red_echo " option must have a git remote to push."
    STATUS=1
  fi
  #test no_remote_set environment variable (must be bool)
  if [ ! "${no_remote_set}" = "true" ] && [ ! "${no_remote_set}" = "false" ];then
    red_echo -n "no_remote_set="
    yellow_echo -n "${no_remote_set}"
    red_echo -n " is not a valid option for no_remote_set!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test authors_file path for existence
  if [ ! -z "${authors_file}" -a ! -f "${authors_file}" ];then
    red_echo -n "Specified "
    yellow_echo -n "--authors-file"
    red_echo " does not exist!"
    STATUS=1
  fi
  #test ssl_verify environment variable (must be bool)
  if [ ! "${ssl_verify}" = "true" ] && [ ! "${ssl_verify}" = "false" ];then
    red_echo -n "ssl_verify="
    yellow_echo -n "${ssl_verify}"
    red_echo -n " is not a valid option for ssl_verify!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test http_remote environment variable (must be bool)
  if [ ! "${http_remote}" = "true" ] && [ ! "${http_remote}" = "false" ];then
    red_echo -n "http_remote="
    yellow_echo -n "${http_remote}"
    red_echo -n " is not a valid option for http_remote!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test enable_colors environment variable (must be bool)
  if [ ! "${enable_colors}" = "true" ] && [ ! "${enable_colors}" = "false" ];then
    red_echo -n "enable_colors="
    yellow_echo -n "${enable_colors}"
    red_echo -n " is not a valid option for enable_colors!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test issues_enabled environment variable (must be bool)
  if [ ! "${issues_enabled}" = "true" ] && [ ! "${issues_enabled}" = "false" ];then
    red_echo -n "issues_enabled="
    yellow_echo -n "${issues_enabled}"
    red_echo -n " is not a valid option for issues_enabled!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test wall_enabled environment variable (must be bool)
  if [ ! "${wall_enabled}" = "true" ] && [ ! "${wall_enabled}" = "false" ];then
    red_echo -n "wall_enabled="
    yellow_echo -n "${wall_enabled}"
    red_echo -n " is not a valid option for wall_enabled!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test wiki_enabled environment variable (must be bool)
  if [ ! "${wiki_enabled}" = "true" ] && [ ! "${wiki_enabled}" = "false" ];then
    red_echo -n "wiki_enabled="
    yellow_echo -n "${wiki_enabled}"
    red_echo -n " is not a valid option for wiki_enabled!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test snippets_enabled environment variable (must be bool)
  if [ ! "${snippets_enabled}" = "true" ] && [ ! "${snippets_enabled}" = "false" ];then
    red_echo -n "snippets_enabled="
    yellow_echo -n "${snippets_enabled}"
    red_echo -n " is not a valid option for snippets_enabled!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test public environment variable (must be bool)
  if [ ! "${public}" = "true" ] && [ ! "${public}" = "false" ];then
    red_echo -n "public="
    yellow_echo -n "${public}"
    red_echo -n " is not a valid option for public!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  #test merge_requests_enabled environment variable (must be bool)
  if [ ! "${merge_requests_enabled}" = "true" ] && [ ! "${merge_requests_enabled}" = "false" ];then
    red_echo -n "merge_requests_enabled="
    yellow_echo -n "${merge_requests_enabled}"
    red_echo -n " is not a valid option for merge_requests_enabled!  Must be "
    yellow_echo -n "true"
    red_echo -n " or "
    yellow_echo -n "false"
    red_echo "."
    STATUS=1
  fi
  return ${STATUS}
}

#
# Main execution
#

#Run a preflight check on options for compatibility.
if ! preflight 1>&2;then
  echo "Command aborted due to previous errors." 1>&2
  exit 1
fi

#Set up project creation options based on config.sh to be passed to create manage_gitlab_project.py
# 构建参数，传递到manage_gitlab_project.py，也就是将shell数据传递到python中
CREATE_OPTS=""
if ${issues_enabled};then
  CREATE_OPTS="--issues ${CREATE_OPTS}"
fi
if ${wall_enabled};then
  CREATE_OPTS="--wall ${CREATE_OPTS}"
fi
if ${merge_requests_enabled};then
  CREATE_OPTS="--merge ${CREATE_OPTS}"
fi
if ${wiki_enabled};then
  CREATE_OPTS="--wiki ${CREATE_OPTS}"
fi
if ${snippets_enabled};then
  CREATE_OPTS="--snippets ${CREATE_OPTS}"
fi
if ${public};then
  CREATE_OPTS="--public ${CREATE_OPTS}"
fi
if ${http_remote};then
  CREATE_OPTS="--http ${CREATE_OPTS}"
fi

# 如果远端不存在，则调用python相应的命令，进行项目的创建
#Get the remote gitlab url for the specified project.
#If the project doesn't already exist in gitlab then create it.
if ! ${no_remote_set} && [ -z "${no_create}" ];then
  green_echo "Resolving gitlab remote." 1>&2
  # 调用python方法，去创建Git仓库
  gitlab_remote=$(python lib/manage_gitlab_project.py --create --groupid ${group_id} --desc "${desc}" ${CREATE_OPTS} "${project_name}")
  # 接收python的返回值，解析为对应的数据
  # retstr=$?
  green_echo ${gitlab_remote}
  # 以&进行字符串拆分，python传出的数据
  array=(${gitlab_remote//,/ })
  # 构建咋们新Git的地址
  gitlab_remote="http://admin_jt:${gitlab_user_token_secret}@192.168.15.20/${array[0]}/${project_name}.git"
  # 同步某个仓库的wiki时，用下面的地址
  # gitlab_remote="http://admin_jt:${gitlab_user_token_secret}@192.168.15.20/${array[0]}/${project_name}.wiki.git"
  # 设置项目的路径名称
  group_name=${array[1]}
  # for var in ${array[@]}
  # do
  #   echo $var
  # done
else
  if ! ${no_remote_set};then
    green_echo -n "Using remote: " 1>&2
    echo "${no_create}" 1>&2
    gitlab_remote="${no_create}"
  else
    echo "Local only mirror." 1>&2
  fi
fi

#Check for namespace directory existence
# 创建项目的Git目录
if [ ! -e "${repo_dir}/${group_name}" ];then
  mkdir -p "${repo_dir}/${group_name}"
elif [ ! -d "${repo_dir}/${group_name}" ];then
  red_echo "Error: \"${repo_dir}/${group_name}\" exists but is not a directory." 1>&2
  exit 1
elif [ -d "${repo_dir}/${group_name}/${project_name}" ] && ! ${force};then
  red_echo "Error: \"${repo_dir}/${group_name}/${project_name}\" exists already.  Aborting command." 1>&2
  exit 1
fi
#Resolve the $authors_file path because of changing working directories
if [ ! -z "${authors_file}" ];then
  if ! echo "${authors_file}" | grep '^/' &> /dev/null;then
    authors_file="${PWD}/${authors_file}"
    authors_file="$(echo ${authors_file} | sed 's#/./#/#g')"
  fi
fi
cd "${git_mirrors_dir}"

if ${git};then
  green_echo "Creating mirror from ${mirror} ${group_name}" 1>&2
  # 进入Git目录
  cd "${repo_dir}/${group_name}"
  # 将老的Git项目克隆到本地
  git clone --mirror "${mirror}" "${project_name}"
  cd "${project_name}"
  if ! ${no_remote_set};then
    green_echo "Adding gitlab remote to project." 1>&2
    # 在本地添加新Git地址为新的远端
    git remote add gitlab "${gitlab_remote}"
    # 配置同步内容
    git config --add remote.gitlab.push '+refs/heads/*:refs/heads/*'
    git config --add remote.gitlab.push '+refs/tags/*:refs/tags/*'
    git config remote.gitlab.mirror true
    #Check the initial repository into gitlab
    green_echo "Checking the mirror into gitlab." 1>&2
    # 从老Git地址拉取最新的内容
    git fetch
    if ${http_remote};then
      git config credential.helper store
    fi
    # 将Git内容，推送到新的远端（新Git地址）
    git push gitlab
    if [ ! -z "${no_create}" ];then
      git config gitlabmirrors.nocreate true
    fi
  else
      git config gitlabmirrors.noremote true
  fi
  green_echo "All done!" 1>&2
fi

四、同步脚本（manage_gitlab_project.py）

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#Created by Sam Gleske
#MIT License
#Created Tue Sep 10 23:01:08 EDT 2013

from sys import argv,exit,stderr
from optparse import OptionParser
import os

try:
  import gitlab
except ImportError:
  raise ImportError("python-gitlab module is not installed.  You probably didn't read the install instructions closely enough.  See docs/prerequisites.md.")

def printErr(message):
  stderr.write(message + "\n")
  stderr.flush()

# 通过组的ID，查找到对应的组位置
def find_group(gid):
  groups = git.groups.list(all=True)

  for obj in groups:
    # printErr(str(obj.id) +" asdasdasd "+ str(gid))
    # printErr(str(obj.web_url))
    if getattr(obj,"id") == int(gid):
      return obj
  return None

# 通过组的名字，和项目名字，确定项目对象
def find_project(gid,pname):
  projects = git.projects.list(as_list=True)
  for obj in projects:
    # printErr(str(getattr(obj,"name")) +"-----"+ str(pname))
    # printErr(str(getattr(obj,"namespace")) +"-----"+ str(gid))
    nspace = getattr(obj,"namespace")
    if getattr(obj,"name") == pname and nspace["id"] == int(gid):
      return obj
  return None

# 创建项目
def createproject(pname):
  if options.public:
     visibility_level="public"
  else:
     visibility_level="private"
  if len(options.desc) == 0:
    if options.public:
      description="Public mirror of %s." % project_name
    else:
      description="Git mirror of %s." % project_name
  else:
    description=options.desc
  project_options={
    'issues_enabled': options.issues,
    'wall_enabled': options.wall,
    'merge_requests_enabled': options.merge,
    'wiki_enabled': options.wiki,
    'snippets_enabled': options.snippets,
    'visibility': visibility_level,
    'namespace_id': options.groupid,
  }
  #make all project options lowercase boolean strings i.e. true instead of True
  for x in project_options.keys():
    project_options[x] = str(project_options[x]).lower()
  printErr("Creating new project %s" % pname)
  project_options['name'] = pname
  project_options['description'] = description
  git.projects.create(project_options)
  found_project = find_project(options.groupid,pname)
  return found_project

# 开始进入游戏主体
try:
  token_secret=os.environ['gitlab_user_token_secret']
  gitlab_url=os.environ['gitlab_url']
  gitlab_user=os.environ['gitlab_user']
  ssl_verify=os.environ['ssl_verify']
  gitlab_api_version=os.environ['gitlab_api_version']
except KeyError:
  printErr("Environment config missing.  Do not run this script standalone.")
  exit(1)
parser = OptionParser()
# 解析路径中的参数
parser.add_option("--issues",dest="issues",action="store_true",default=False)
parser.add_option("--wall",dest="wall",action="store_true",default=False)
parser.add_option("--merge",dest="merge",action="store_true",default=False)
parser.add_option("--wiki",dest="wiki",action="store_true",default=False)
parser.add_option("--snippets",dest="snippets",action="store_true",default=False)
parser.add_option("--public",dest="public",action="store_true",default=False)
parser.add_option("--create",dest="create",action="store_true",default=False)
parser.add_option("--delete",dest="delete",action="store_true",default=False)
parser.add_option("--desc",dest="desc",metavar="DESC",default=False)
parser.add_option("--groupid",dest="groupid",metavar="GROUPID",default=False)
parser.add_option("--http",dest="http",action="store_true",default=False)
(options,args) = parser.parse_args()
# printErr(str(options)+"   ajskldkljajklf")

if len(args) == 0:
  printErr("No project name specified.  Do not run this script standalone.")
  exit(1)
elif len(args) > 1:
  printErr("Too many arguments.  Do not run this script standalone.")
  exit(1)

project_name=args[0]

# 链接到git仓库
if not eval(ssl_verify.capitalize()):
  git=gitlab.Gitlab(gitlab_url,token_secret,ssl_verify=False,api_version=gitlab_api_version)
else:
  git=gitlab.Gitlab(gitlab_url,token_secret,ssl_verify=True,api_version=gitlab_api_version)

if options.create:
  found_group = find_group(options.groupid)
  # 打印git组信息
  # printErr(str(found_group))

  found_project = None
  # 查找项目是否存在
  found_project= find_project(options.groupid,project_name)
  if not found_project:
    # 项目不存在的情况，则创建项目
    found_project=createproject(project_name)
    if not found_project:
      printErr("There was a problem creating {group}/{project}.  Did you give {user} user Admin rights in gitlab?".format(group=found_group.name,project=project_name,user=gitlab_user))
      exit(1)
  # 打印项目的结构体
  # printErr(str(found_project))
  # 下方的print是为了让shell能接收到返回的数据
  print(found_group.full_path+","+found_project.name)
  if options.http:
    print(found_project.http_url_to_repo)
  else:
    print(found_project.ssh_url_to_repo)
elif options.delete:
  # 执行项目的删除
  try:
    deleted_project=find_project(options.groupid,project_name).delete()
  except Exception as e:
    printErr(e)
    exit(1)
else:
  printErr("No --create or --delete option added.")
  exit(1)

五、运行命令