以多进程/线程方式执行命令

import queue
import threading
import multiprocessing
import time
from datetime import datetime
import random
from collections import OrderedDict


class ExecFlowSeq(object):
    def __init__(self):
        pass

    def func_process(self, item: str):
        pass

    def func_multiprocess(self, item: str):
        status = "Successed"
        error_info = ""
        try:
            dt_start = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
            # process with item
            self.func_process(item)
            time.sleep(1)
        except Exception as err:
            status = "Failed"
            error_info = err
        finally:
            dt_finish = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
            result = {"item": item, "dt_start": dt_start, "dt_finish": dt_finish, "status": status,
                      "error_info": error_info}
            print(result)
        return result

    def multiprocess_items(self, flow_seq: OrderedDict):
        '''
        :param flow_seq: OrderDict({"key":list})
        :param func_mp: funcion
        :return:
        '''
        res = []
        for step, item_list in flow_seq.items():
            # load_set is a list,tasks will be paralle
            pool_num = min(20, multiprocessing.cpu_count())
            pool = multiprocessing.Pool(pool_num)
            for index, item_str in enumerate(item_list):
                res.append(pool.apply_async(func=self.func_multiprocess, args=(item_str,)))
            pool.close()
            pool.join()
        return [r.get() for r in res]

    def func_thread(self, item: str, queue: queue.Queue = None):
        status = "Successed"
        error_info = ""
        try:
            dt_start = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
            self.func_process(item)
            time.sleep(1)
        except Exception as err:
            status = "Failed"
            error_info = err
        finally:
            dt_finish = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
            result = {"load_item": item, "dt_start": dt_start, "dt_finish": dt_finish, "status": status,
                      "error_info": error_info}
            if queue is not None:
                queue.put(result)
            print(result)
        return result

    def thread_items(self, flow_seq: OrderedDict):
        q = queue.Queue()
        threads = []
        for k,load_set in flow_seq.items():
            for load_item in load_set:
                t = threading.Thread(target=self.func_thread, args=[load_item, q])
                t.start()
                threads.append(t)
            for t in threads:
                t.join()
        while not q.empty():
            result = q.get()
            if result["status"] == "Failed":
                raise Exception("Thread Failed")


if __name__ == '__main__':
    flow_seq = OrderedDict()
    for i in range(3):
        load_set = []
        for j in range(1, int(random.random() * 6) + 1):
            load_item = 'abcdefghijklmnopqrstuvwxyz'[int(random.random() * 26)]
            load_set.append(load_item)
        flow_seq[i + 1] = load_set
    print(flow_seq)
    # OrderedDict([(1, ['a', 'e', 'b', 'a', 'b']), (2, ['g', 'r', 'h', 'r']), (3, ['e'])])
    # 同一个子列表里的内容并行，不同级别之间串行
    t0 = time.time()
    EFS = ExecFlowSeq()
    rs = EFS.multiprocess_items(flow_seq)
    t1 = time.time()
    print("ttl time cost with multiprocess {:.2f}s".format(t1 - t0))
    EFS.thread_items(flow_seq)
    print("ttl time cost with thread {:.2f}s".format(time.time() - t1))