python的multiprocessing

创建多个进程

import multiprocessing
from multiprocessing import Process, Lock
import logging
import os

logger = multiprocessing.log_to_stderr() #打印日志
logger.setLevel(multiprocessing.SUBWARNING)
thread_count = 5

def worker(lock,i):
        lock.acquire()
        print str(i)+aaa
        print('module name:', __name__)
        if hasattr(os, 'getppid'):  # only available on Unix
                print('parent process:', os.getppid())
        print('process id:', os.getpid())
        logger.warning('doomed')
        lock.release()

if __name__ == '__main__':
        lock = Lock()
        for i in range(thread_count):
                Process(target=worker, args=(lock,i,)).start()

multiprocessing 提供 Queue 与 Pipe两种方式传输数据

#Queue
from multiprocessing import Process, Queue

def f(q):
    q.put([42, None, 'hello'])

if __name__ == '__main__':
    q = Queue()
    p = Process(target=f, args=(q,))
    p.start()
    print(q.get())    # prints "[42, None, 'hello']"
    p.join()

 
#Pipe 
from multiprocessing import Process, Pipe

def f(conn):
    conn.send([42, None, 'hello'])
    conn.close()

if __name__ == '__main__':
    parent_conn, child_conn = Pipe()
    p = Process(target=f, args=(child_conn,))
    p.start()
    print(parent_conn.recv())   # prints "[42, None, 'hello']"
    p.join()

共享内存传递数据结果,Value和Array都是线程安全    

from multiprocessing import Process, Value, Array

def f(n, a):
    n.value = 3.1415927
    for i in range(len(a)):
        a[i] = -a[i]

if __name__ == '__main__':
    num = Value('d', 0.0)
    arr = Array('i', range(10))

    print(num.value) # 0.0
    print(arr[:])    # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
    p = Process(target=f, args=(num, arr))
    p.start()
    p.join()

    print(num.value) # 3.1415927
    print(arr[:])    # [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]

 进程池

import multiprocessing
import time

def do_calculation(data):
    print(multiprocessing.current_process().name + " " + str(data))
    time.sleep(3)
    return data * 2

def start_process():
    print ('Starting', multiprocessing.current_process().name)

if __name__ == '__main__':
    inputs = list(range(10))
    print ('Input   :' + str(inputs))

    pool_size = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(processes=pool_size,
                                initializer=start_process,
                                )
    pool_outputs = pool.map(do_calculation, inputs)
    pool.close() # no more tasks
    pool.join()  # wrap up current tasks

    print ('Pool    :' + str(pool_outputs))

 输出如下

root:~ # python f.py
Input   :[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
('Starting', 'PoolWorker-1')
PoolWorker-1 0
('Starting', 'PoolWorker-2')
PoolWorker-2 2
PoolWorker-1 1
PoolWorker-2 3
PoolWorker-1 4
PoolWorker-2 6
PoolWorker-1 5
PoolWorker-2 7
PoolWorker-1 8
PoolWorker-1 9
Pool    :[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]