python多线程multiprocessing

Pool

例1：

import multiprocessing
import os

def info(title):
    print title
    print 'module name:',__name__
    if hasattr(os, 'getppid'):
        print 'parent process:', os.getppid()
    
    print 'process id:', os.getpid()
    
def f(name):
    info('function f')
    print 'hello', name
    
if __name__ == '__main__':
    info('main line')
    p = multiprocessing.Pool(5)
    p.map(f, ['bob','lily'])

输出：

main line
module name: __main__
parent process: 29570
process id: 30583

function f
module name: __main__
parent process: 30583
process id: 30590
hello bob

function f
module name: __main__
parent process: 30583
process id: 30589
hello lily

其中：

进程29570为eclipse，其子进程30583为主进程，进程30590，30589为主进程30583创建的子进程。

CPU测试：

import multiprocessing
    
def f(name):
    print name
    while True:
        pass
    
if __name__ == '__main__':
    p = multiprocessing.Pool(2)
    p.map(f,['bob','lily'])

结果：

两个子进程分配到了两个核上。

例2：

from multiprocessing import Pool

def f(x):
    return x*x

if __name__ == '__main__':
    pool = Pool(processes = 4)
    rst = pool.apply(f, [10])
    print rst   # 100
    
    # apply的实现
    # def apply(self, func, args=(), kwds={}):
    #     '''
    #     Equivalent of `apply()` builtin
    #     '''
    #     assert self._state == RUN
    #     return self.apply_async(func, args, kwds).get()
    
    rst = pool.apply_async(f, [10])
    print rst.get() #100

IPC(进程间通信)

Queue

from multiprocessing import Process, Queue
import time


def f(q):
    time.sleep(5)
    q.put('Bob')


if __name__ == "__main__":
    q = Queue()
    p = Process(target=f, args=(q,))
    p.start()
    n = q.get()         #此处会阻塞5秒
    print 'hello,',n

Pipe

from multiprocessing import Process, Pipe
import time

def f(conn):
    time.sleep(5)
    conn.send(["hello","bob"])
    conn.close()

if __name__ == "__main__":
    parent_conn, child_conn = Pipe()
    p = Process(target=f, args=(child_conn,))
    p.start()
    msg = parent_conn.recv()    #此处阻塞5秒
    print msg

Pipe由一对线程共享，Queue由多个线程共享。

进程同步：Lock

不加锁：

from multiprocessing import Process, Lock
import time

def f(lock, i):
    time.sleep(1)
    print i,
    print '-------',
    print i

if __name__ == '__main__':
    lock = Lock()

    for num in range(10):
        Process(target=f, args=(lock, num)).start()

输出：

01  --------------  10

2 ------- 2
43 ------- 4
 ------- 3
56  --------------  65

7 ------- 7
89  --------------  98

加锁：

from multiprocessing import Process, Lock
import time

def f(lock, i):
    time.sleep(1)
    lock.acquire()
    print i,
    print '-------',
    print i
    lock.release()

if __name__ == '__main__':
    lock = Lock()

    for num in range(10):
        Process(target=f, args=(lock, num)).start()

输出：

0 ------- 0
1 ------- 1
2 ------- 2
3 ------- 3
4 ------- 4
5 ------- 5
6 ------- 6
7 ------- 7
8 ------- 8
9 ------- 9