针对"因PG和PGCACHE共用镜像，若推库有时间差可导致整系统CI失败”的问题解决流程图

前提条件：

pg和pgcache，共用server和broker的镜像，pg的三个蓝图版本号和镜像的版本号一致，pgcache的三个蓝图版本号与镜像的版本号不一致，规律如下。

举例：（读者如果需要用到这个规律，最好再去跟PG的同事确认一下以免有更改）

       10.6.0.9.3.xxx   pg

       10.6.0.9.4.xxx   pgcache

       3和4是固定的，3是pg,4是pgcache

说明：之所以在PG推前置库的时候使用version_data_pgcache.txt这个文件记录此时PGCACHE的版本号，是为了应对如下场景。举例，前置库中目前只有v1.0的PG和PGCACHE，现在推送了v1.1的PG，触发源检测到了PG版本变化，它的version_data.txt已经被新的版本号覆盖，新PG开始跑ST用例，ST用例大概要跑30分钟，在此期间，v1.1版本的PGCACHE也推到了前置库，因为触发源和ST用例是两套Jenkins，所以触发源依然在定时去查有无新版本的公用服务推入前置库，此时，PGCACHE的version_data.txt内容也被新的版本号覆盖，但是PG用例跑完之后要去比对PG和PGCACHE的version_data.txt的内容是否一致，如果两者一致就直接把v1.1版本的PG和PGCACHE共同推项目库的话，此时v1.1的PGCACHE是还没开始跑ST用例的，肯定不能推项目库，所以就要有另一个条件来约束，这就是为什么在v1.1的PG触发了触发源的时候就应该把那个时间点的PGCACHE最新版本记录下来并缓存。

流程图如下：

Jenkins_1实现的是触发源。

Jenkins_2实现的是跑ST用例并且把公用服务的镜像、蓝图推到项目库的一整套流程。

异常场景的保护机制

场景描述：

如果PG、PGCache跑用例失败，如果按照之前脚本文件的逻辑，两者都不会上传，但是，当PG先推库，没有跑过用例，此时再推PGCache，它跑过了用例，那么按照文中设计的流程，PGCache这条线会直接把PG和PGCache都推到项目库，这就会导致有问题的PG虽然没有跑过用例，但是它却被推进了项目库。

应对策略：

PG和PGCache各自对应的cs-actest.sh代码中，当用例跑失败时，增加一个操作就是将对应的version_data.txt文件内容清空，这就避免了上述问题。

Jenkins_1的代码（触发源）

Jenkins_1，触发源的python代码实现

import os
import commands
import subprocess
from subprocess import call
from subprocess import Popen
import paramiko
import json
import requests
import time
import sys
import simplejson
import CSACGlobalVar

controllerIP=CSACGlobalVar.controllerIP
username=CSACGlobalVar.username
password=CSACGlobalVar.password
deletedstring=CSACGlobalVar.deletedstring

synclistsourcezart=CSACGlobalVar.synclistsourcezart

synclistsourceregistry=CSACGlobalVar.synclistsourceregistry

synclistname=CSACGlobalVar.synclistname

zartcli=CSACGlobalVar.csprezart

def get_externalimageversion(imagename=""):
while(True):
cmd=“sshpass -p %s ssh -o StrictHostKeyChecking=no %s@%s
-tt “sudo /root/zartcli/zartcli -o=query -i=admin -m=image -n=%s -s %s”” % (password,username,controllerIP,imagename,zartcli)
status,output=commands.getstatusoutput(cmd)
list = output.split(’--------’)
list = list[-1].replace(deletedstring,’’,1)
externalVersion = json.loads(list)
externalVersion = externalVersion[‘list’][0][‘tags’]
if externalVersion[-1][“status”] == ‘available’:
print ‘-’*60,‘external image available’,’-’*60
print externalVersion[-1]
break
# externalversions=[]
# for i in xrange(len(externalVersion)):
# externalversions.append(externalVersion[i][‘version’])
externalversions = externalVersion[-1][‘version’]
return externalversions

#judge external version if or not exist in local version
def judge_ExternalVersion_ExistInLocal(externalversion,localversions):
print ‘The external version:\n’,externalversion,’\n’
print ‘The local version:\n’,localversions,’\n’
if externalversion in localversions:
print ‘External version already exist.\n’
return False
else:
print ‘External version not exist.\n’
return True

def get_CommonServiceVersion_ByName(csname=""):
check_microservice_or_service(csname)
csdir=CSACGlobalVar.CommonServiceList[csname]

# csdir['image']=[csdir['image'],get_externalimageversion(csdir["image"])[-1]]
csdir = get_externalimageversion(csdir["image"])
print "\nGet_CommonServiceVersion_ByName:\n",csdir,'\n'

return csdir

def check_microservice_or_service(csname=""):
if csname==‘Apache-Kafka’ or csname == ‘Redis-inner-HA’:
global tag
tag=‘service’
else:
global tag
tag=‘microservice’

def judge_VersionsSame(versiondir):
version_value=str(versiondir)
file1 = open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQLCACHE_VNPM/version_data.txt", “r”)
ver = file1.read()
file1.close()
if ver != version_value:
with open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQLCACHE_VNPM/version_data.txt", “w”) as file2:
file2.write(version_value)
return False
else:
return True

def record_pg_version():
file1 = open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQL_VNPM/version_data.txt", “r”)
ver = file1.read()
file1.close()
with open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQLCACHE_VNPM/version_data_pg.txt", “w”) as file2:
file2.write(ver)

if name==“main”:
time.sleep(1)
# Check commonservice push images finished.
versiondir=get_CommonServiceVersion_ByName(sys.argv[1])

checkresult = judge_VersionsSame(versiondir)
if not checkresult:
    record_pg_version()
    print('call another jenkins')
    # n=os.system('/home/zte/jenkins/workspace/CommonServicePipeLine/call_jenkins.sh')
    # exit(1)
else:
    raise Exception

Jenkins_2的核心代码

Jenkins_2中的推库部分的python代码实现

import os
import commands
import subprocess
from subprocess import call
from subprocess import Popen
import paramiko
import json
import re
import time
import sys
import simplejson
import CSACGlobalVar
import CSACGetVersion

controllerIP = CSACGlobalVar.controllerIP
username = CSACGlobalVar.username
password = CSACGlobalVar.password
deletedstring = CSACGlobalVar.deletedstring
projectzart = CSACGlobalVar.projectzart
syncname = ‘’

def execute_synclistfile(zartcli):
print ‘-’ * 70, ‘starting sync’, ‘-’ * 70
cmd = “sshpass -p %s ssh -o StrictHostKeyChecking=no %s@%s
-tt “sudo /root/zartcli/zartcli -o=sync -i=admin -p=/root/zartcli -s %s”” % (
CSACGlobalVar.password, CSACGlobalVar.username, CSACGlobalVar.controllerIP, zartcli)
status, output = commands.getstatusoutput(cmd)
if status == 0:
print output, ‘\n’, ‘-’ * 70, ‘sync success’, ‘-’ * 70
else:
print output, ‘\n’, ‘-’ * 70, ‘sync failed’, ‘-’ * 70

def get_sync(model=’’, syncname=’’, ip=’’):
while (True):
cmd = “sshpass -p %s ssh -o StrictHostKeyChecking=no %s@%s
-tt “curl http://%s/swr/v1/tenants/admin/%s/sync/%s”” % (
CSACGlobalVar.password, CSACGlobalVar.username, CSACGlobalVar.controllerIP, ip, model, syncname)
print cmd
status, output = commands.getstatusoutput(cmd)
print output
list = output.split(’--------’)
list = list[-1].replace(deletedstring, ‘’, 1)
list = json.loads(list)
if list == []:
return False
# if list[0][‘donetask’]==list[0][‘totaltask’]:
if list[0][‘status’] == ‘sync success’:
print ‘-’ * 70, ‘%s Sync successed’ % model, ‘-’ * 70
return True
elif list[0][‘status’] == ‘sync fail’:
print ‘-’ * 70, ‘%s Sync failed’ % model, ‘-’ * 70
exit(1)
else:
print ‘-’ * 70, ‘%s pending or doing sync’ % model, ‘-’ * 70
time.sleep(10)
continue
# else:
# print ‘-’*70,’%s Donetask is not equal to totaltask!’ % model,’-’*70
# exit(1)

def get_synclist(ip=’’):
cmd = “sshpass -p %s ssh -o StrictHostKeyChecking=no %s@%s
-tt “cat /root/zartcli/sync.list”” % (
CSACGlobalVar.password, CSACGlobalVar.username, CSACGlobalVar.controllerIP)
status, output = commands.getstatusoutput(cmd)
print output
list = output.split(’--------’)
list = list[-1].replace(deletedstring, ‘’, 1)
list = json.loads(list)
print list
print list[‘name’]
syncname = list[‘name’]

get_sync('bps', syncname, ip)
get_sync('images', syncname, ip)
get_sync('coms', syncname, ip)

def remove_synclistfile():
cmd = “sshpass -p %s ssh -o StrictHostKeyChecking=no %s@%s
-tt “sudo rm /root/zartcli/sync.list”” % (
CSACGlobalVar.password, CSACGlobalVar.username, CSACGlobalVar.controllerIP)
commands.getstatusoutput(cmd)

def scp_synclistfile(synclistcontent):
fi = open("/home/zte/jenkins/workspace/sync_pg.list", “w”)
fi.write(synclistcontent)
fi.close()
cmd = “sshpass -p %s ssh -o StrictHostKeyChecking=no %s@%s
-tt “sudo sshpass -p ‘XXXXXXXXXXXXX’ scp root@XX.XX.XXX.XXX:/home/zte/jenkins/workspace/sync.list /root/zartcli”” % (
CSACGlobalVar.password, CSACGlobalVar.username, CSACGlobalVar.controllerIP)
status, output = commands.getstatusoutput(cmd)
print status, output

def synclist_add_pg_or_pgcache(srvname=’’, version_present=’’):
if srvname == ‘PostgreSQL_VNPM’:
add_list = [{“model”: “bp”, “version”: version_present, “reponame”: “admin”, “name”: “commsrv_pgcache_vnpm_bp”},
{“model”: “bp”, “version”: version_present, “reponame”: “admin”, “name”: “commsrv_pgcache_vnpm_brokerbp”},
{“model”: “bp”, “version”: version_present, “reponame”: “admin”, “name”: “commsrv_pgcache_vnpm_srvbp”}]

if srvname == 'PostgreSQLCACHE_VNPM':
    add_list = [{"model": "bp", "version": version_present, "reponame": "admin", "name": "commsrv_pg_vnpm_bp"},
                {"model": "bp", "version": version_present, "reponame": "admin", "name": "commsrv_pg_vnpm_brokerbp"},
                {"model": "bp", "version": version_present, "reponame": "admin", "name": "commsrv_pg_vnpm_srvbp"}]
file = open("/home/zte/jenkins/workspace/sync_pg.list", "r")
ver = file.read()
file.close()
olb_sync_list = json.loads(ver)
# print olb_sync_list

olb_sync_list['versions'].extend(add_list)
synclist = json.dumps(olb_sync_list)

scp_synclistfile(synclist)
print synclist

if name == “main”:
a = sys.argv[1]
# a = ‘PostgreSQL_VNPM’
if a == ‘PostgreSQL_VNPM’:
file1 = open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQL_VNPM/version_data.txt", “r”)
ver1 = file1.read()
file1.close()

    file2 = open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQLCACHE_VNPM/version_data.txt", "r")
    ver2 = file2.read()
    file2.close()

    file3 = open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQL_VNPM/version_data_pgcache.txt", "r")
    ver3 = file3.read()
    file3.close()

    if ver1 != ver2:
        # with open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQL_VNPM/version_data.txt", "w") as file2:
        #     file2.write(version_value)
        print '-' * 70, 'End: %s result' % a, '-' * 70

        print "Note: PostgreSQLCACHE_VNPM hasn't been pushed yet, please push it immediately !!!!!!"
    elif ver1 != ver3:
        print '-' * 70, 'End: %s result' % a, '-' * 70

        print "Note: PostgreSQLCACHE_VNPM has been pushed yet, please wait the success " \
              "of the jenkins which test PostgreSQLCACHE_VNPM!!!!!!"
    else:
        synclist_add_pg_or_pgcache(a, ver1)
        execute_synclistfile('r9')
        get_synclist(CSACGlobalVar.projectzart)
        remove_synclistfile()

elif a == 'PostgreSQLCACHE_VNPM':
    file1 = open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQL_VNPM/version_data.txt", "r")
    ver1 = file1.read()
    file1.close()

    file2 = open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQLCACHE_VNPM/version_data.txt", "r")
    ver2 = file2.read()
    file2.close()

    file3 = open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQL_VNPM/version_data_pgcache.txt", "r")
    ver3 = file3.read()
    file3.close()
    if ver1 != ver2:
        # with open("/home/zte/jenkins/workspace/common_srv_sdev/sdev_PostgreSQL_VNPM/version_data.txt", "w") as file2:
        #     file2.write(version_value)
        print '-' * 70, 'End: %s result' % a, '-' * 70
        print "Note: PostgreSQL_VNPM hasn't been pushed yet, please push it immediately !!!!!!"
    elif ver1 != ver3:
        print '-' * 70, 'End: %s result' % a, '-' * 70

        print "Note: PostgreSQLCACHE_VNPM has been pushed yet, please wait the success " \
              "of the jenkins which test PostgreSQL_VNPM!!!!!!"
    else:
        synclist_add_pg_or_pgcache(a, ver1)
        execute_synclistfile('r9')
        get_synclist(CSACGlobalVar.projectzart)
        remove_synclistfile()

else:
    execute_synclistfile('r9')
    get_synclist(CSACGlobalVar.projectzart)
    remove_synclistfile()