python(18): 线程池，多线程，子进程多层调用实践

一.需求

 

多个镜像仓库之间的镜像同步：根据接收到的指令数据，开始镜像同步程序进行同步并采集分析同步结果，上传同步结果

备注：镜像同步时需要占用磁盘IO和网络IO资源，所以需要限制并发执行数

二.方案图

描述：

1.采用线程池方式控制同时执行的线程数

2.消息队列实现任务的发送和接收

3.子进程实现镜像同步工具的调用，

4.线程池某个线程中再开启两个线程实现对镜像同步工具stdout,stderr信息的接收

5.根据镜像同步工具输出规律，正则表达式实现对输出结果分析出成果/失败

图：

三.部分代码

父类：抽象

class Task():
    '''
    异步任务基类
    status: waiting，running，succeed , failed
    '''
    STATUS = ('waiting','running','succeed','failed')
    def __init__(self,params):
        self.id = params['taskid']
        self.result = None
        self.status = None

    def run(self,pool):
        self.future = pool.submit(self._run)
        self.future.add_done_callback(thread_result_callback)

    def get_result(self):
        self.result = self.future.result()
        return self.result

    def stop(self):
        pass

    def _run(self):
        pass

 实现具体功能类

class ImageAsynTask(Task):
    def __init__(self,params):
        super(ImageAsynTask,self).__init__(params)
        self._params = params
        self.stdout = ''
        self.stderr = ''
        self.output = ''
        tool = os.path.join(project_path,'tool/image-syncer')
        self.command = tool + ' --config '
        self.timeout = 300 if not params.get('timeout') else int(params.get('timeout'))
        self.msg = ''

    def _run(self):
        try:
            update_task_status(self.id, self.STATUS[1])  # 更新数据库任务状态
            configfile = self.buile_configfile()
            cmd = self.command + configfile
            subpro = subprocess.Popen(cmd, shell=True,stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            threads = []
            threads.append(threading.Thread(target=self.read_stdout_to_msg, args=(subpro,)))
            threads.append(threading.Thread(target=self.read_stderr_to_msg, args=(subpro,)))
            for item in threads:
                item.start()
            # for item in threads:
            #     item.join()
            subpro.wait(self.timeout)  # 进程提前结束会提前执行通过，任务超时会报异常

            # if len(self.stderr) > 0:   #不能用此方法判断：正常运行时程序部分输出到stderr
            #     self.status = self.STATUS[3]
            self.status = self.STATUS[2]
            if os.path.exists(configfile):
                os.remove(configfile)
            self.analysis_result()  # 根据输出分析执行结果，进一步确定任务执行是否成功
        except Exception as e:
            print(e)
            self.msg = str(e)
            self.status = self.STATUS[3]
        finally:
            return {'result': self.status, 'msg': self.msg, 'taskid': self.id, 'output': self.output}

    def read_stdout_to_msg(self,process):
        '''
        读取进程stdout信息到msg
        :param process:
        :param msg:
        :return:
        '''
        process_status = process.poll()
        while not process_status:
            line = process.stdout.readline()
            if len(line) == 0:
                if process.poll() == None:
                    try:
                        process.terminate()
                    except Exception as e:
                        pass
                break

            line = str(convert_encode_to_utf8(line),encoding='utf-8')
            self.stdout = self.stdout + line
            self.output = self.output + line
            process_status = process.poll()
        print('stdout=======end')

    def read_stderr_to_msg(self,process):
        '''
        读取进程stderr信息到msg
        :param process:
        :param msg:
        :return:
        '''
        process_status = process.poll()
        while not process_status:
            line = process.stderr.read()
            if len(line) == 0:
                if process.poll() == None:
                    try:
                        process.terminate()
                    except Exception as e:
                        pass
                break
            line = str(convert_encode_to_utf8(line), encoding='utf-8')
            self.stderr = self.stderr + line
            self.output = self.output + line
            process_status = process.poll()
        print('stderr=======end')

    def buile_configfile(self):
        '''
        根据参数创建image-syncer  config.js文件
        :param params:
        :return:  文件路径
        '''
        params = self._params
        source_region = params['source_region']
        source_product = params['source_product']
        source_version = params['source_version'] + '/'
        dst_version = params['dst_version'] + '/'
        dst_product = params['dst_product']
        dst_region = params['dst_region']

        src_images = []  #存放最近推送的镜像
        image_names = registry_client.list_repositories(source_product+'/'+source_version)
        for image in image_names:
            tagslist = registry_client.sort_taglist(image)
            if tagslist['tags'] != []:
                src_images.append(APPLICATION_CONFIG["REGISTRY_ENDPOINT"]+'/'+image+':'+tagslist['tags'][0])

        if source_region == dst_region:
            dst_server_data={'account':APPLICATION_CONFIG["REGISTRY_USERNAME"],'password':APPLICATION_CONFIG["REGISTRY_PASSWORD"],'insecure':APPLICATION_CONFIG["REGISTRY_INSECURE"],'type_value':APPLICATION_CONFIG["REGISTRY_ENDPOINT"]}
        else:
            dst_server_data = get_server_data_from_db(dst_region, 'docker')
        dst_server_endpoint = dst_server_data.get('type_value')

        config = {"auth":{
                        APPLICATION_CONFIG["REGISTRY_ENDPOINT"]:{
                            "username": APPLICATION_CONFIG["REGISTRY_USERNAME"],
                            "password": APPLICATION_CONFIG["REGISTRY_PASSWORD"],
                            "insecure": APPLICATION_CONFIG["REGISTRY_INSECURE"]
                                                                },
                       dst_server_endpoint:{
                           "username": dst_server_data.get('account'),
                           "password": dst_server_data.get('password'),
                           "insecure": dst_server_data.get('insecure')
                       }},
            "images":{}}

        for image in src_images:
            image_name = image.split('/',3)[-1]
            dst_image = dst_server_endpoint + '/'+ dst_product + '/' + dst_version + image_name
            config["images"].update({image:dst_image})
        #生成配置文件
        filename = os.path.join(project_path,'tool/config','config_{}.json'.format(params['taskid']))
        with open(filename,mode='w+') as f:
            f.write(json.dumps(config))
        return filename

    def analysis_result(self):
        '''
        根据执行输出分析执行结果
        :return:
        '''
        if self.status == self.STATUS[3] or len(self.output) == 0:
            return
        result_str = self.output.strip().split('\n')[-1]
        if 'Finished' not in result_str:
            self.status == self.STATUS[3]
            return
        result = re.findall(r'(\d{1,10}) sync tasks failed, (\d{1,10}) tasks generate failed',result_str)
        if len(result) == 0 or len(result[0]) < 2:
            self.status = self.STATUS[3]
            return
        if result[0][0] == '0' and result[0][1] == '0':
            self.status = self.STATUS[2]
        else:
            self.status = self.STATUS[3]

 程序入口

if __name__ == '__main__':
    workers = APPLICATION_CONFIG['QUEUE_NAME']['FILEASYN_WOKERS']
    with ThreadPoolExecutor(max_workers=workers) as pool:  # 创建一个最大容纳数量为5的线程池
        while(True):
            try:
                remsg = redisclient.brpop(APPLICATION_CONFIG['QUEUE_NAME']['ASYN_TASK'])   #长时间空闲阻塞，服务端可能会断开连接
            except Exception as e:
                print(e)
            values = json.loads(remsg['values'])
            task = build_task(values)
            task.run(pool)
            # future = pool.submit(file_asyn_task, taskid, params)
            # future.add_done_callback(get_thread_result)
            print('{0}: add thread ; msg: {1}'.format(datetime.now(),remsg))