多线程多进程运行python任务

1.多进程

from multiprocessing import Manager, Pool
import time
import os
import subprocess
from subprocess import PIPE

def throw_exception(name):
    print('子进程%s发生异常,进程号为%s'%(name,os.getpid()))
    cmd = 'taskkill /im '+ str(os.getpid())+' /F'
    res = subprocess.Popen(cmd,shell=True,stdin=PIPE,stdout=PIPE)
    print(res.stdout.read())
    print(res.stderr.read())
    time.sleep(2)

def one_core(my_func, query_id, special_input, share_input, result=None):
    if result is None:
        result = {}
    func_res = my_func(special_input,share_input)
    result[query_id] = func_res
    return result

def run_multi_processing(my_func, input_dict, share_input, threading_num=1):
    result = Manager().dict()
    pool = Pool(processes=threading_num if threading_num >= 8 else threading_num)
    for query_id in input_dict:
        pool.apply_async(one_core, args = (my_func, query_id, input_dict[query_id], share_input, result),error_callback=throw_exception)
    pool.close()
    pool.join()
    return result

2.多线程

import threading

class PathThreading(threading.Thread):
    def __init__(self, threadID, name, my_func, special_input, share_input):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.result_dict = None
        self.my_func = my_func
        self.special_input = special_input
        self.share_input = share_input

    def run(self):
        print("开始线程：" + self.name)
        self.result_dict = self.my_func(self.special_input, self.share_input)
        print("退出线程：" + self.name)


def run_multi_thread(my_func, input_dict, share_input, threading_num=1):
    if threading_num == 1:
        my_func(input_dict, share_input)
    else:
        thread_list = []
        for index in range(threading_num):
            special_input_dict = {k: input_dict[k] for i, k in enumerate(input_dict) if i % threading_num == index}
            thread_list.append(PathThreading(index, "Thread " + str(index), my_func, special_input_dict, share_input))

        for thread in thread_list:
            thread.start()
        for thread in thread_list:
            thread.join()

        overall_result_dict = {}
        for t in thread_list:
            overall_result_dict.update(t.result_dict)
        return overall_result_dict