python-multiprocessing简单应用

参考：https://www.liaoxuefeng.com/wiki/1016959663602400/1017628290184064

入门例子

from multiprocessing import Process
import os

def run_proc(name):
    print("Run child process %s(%s)"%(name, os.getpid()))

if __name__ == "__main__":
    print("Parent process %s"%os.getpid())
    p = Process(target=run_proc, args=('test',))
    print("Child process will start")
    p.start()
    p.join() # 等待子进程结束后再继续往下运行，通常用于进程之间的同步
    print("Child process end")
    '''
    Parent process 7368
    Child process will start
    Run child process test(19684)
    Child process end
    '''

进程池：批量创建子进程

from multiprocessing import Pool
import os, time, random

def long_time_task(name):
    print('Run task %s (%s)...' % (name, os.getpid()))
    start = time.time()
    time.sleep(random.random() * 3)
    end = time.time()
    print('Task %s runs %0.2f seconds.' % (name, (end - start)))

if __name__=='__main__':
    print('Parent process %s.' % os.getpid())
    p = Pool(5)
    for i in range(5):
        p.apply_async(long_time_task, args=(i,))
    print('Waiting for all subprocesses done...')
    p.close()
    p.join()
    print('All subprocesses done.')
'''
Parent process 12260.
Waiting for all subprocesses done...
Run task 0 (5540)...
Run task 1 (5940)...
Run task 2 (4408)...
Run task 3 (15124)...
Run task 4 (20676)...
Task 4 runs 0.44 seconds.
Task 3 runs 0.53 seconds.
Task 1 runs 1.11 seconds.
Task 0 runs 1.28 seconds.
Task 2 runs 1.79 seconds.
All subprocesses done.
'''

在Unix/Linux下，可以使用fork()调用实现多进程。

要实现跨平台的多进程，可以使用multiprocessing模块。

进程间通信是通过Queue、Pipes等实现的。

一个实例

# process和timeout都是前面指定的参数
processes = multiprocessing.cpu_count() if processes < 1 else processes
timeout = None if timeout < 1 else timeout

with multiprocessing.Pool(processes=processes) as pool:
    for i, gene_seq in enumerate(gene_seqs):
    	# func是我的计算函数
        res = pool.apply_async(func, (gene_seq,))
        try:
        	# 获取返回值
            predict, message = res.get(timeout=timeout)
            predicts[i] = predict
            messages[i] = message
        except multiprocessing.TimeoutError:
            predicts[i] = -5
            messages[i] = 'PredictTimeoutError'
        except Exception:
            predicts[i] = -6
            messages[i] = 'PredictUnknownError'

    pool.close()
    pool.join()