Python高性能编程

 一、进程池和线程池

1.串行

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22

import time import requests url_lists = [     'http://www.baidu.com',     'http://fanyi.baidu.com',     'http://map.baidu.com',     'http://music.baidu.com/',     'http://tieba.baidu.com',     'http://v.baidu.com',     'http://image.baidu.com',     'http://zhidao.baidu.com',     'http://news.baidu.com',     'http://xueshu.baidu.com'] start_time = time.time() for url in url_lists:     response = requests.get(url)     print(response.text) print("Runtime: {}".format(time.time()-start_time)) # Runtime: 1.95

　　

2.多进程

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34

import time import requests from multiprocessing import Process url_lists = [     'http://www.baidu.com',     'http://fanyi.baidu.com',     'http://map.baidu.com',     'http://music.baidu.com/',     'http://tieba.baidu.com',     'http://v.baidu.com',     'http://image.baidu.com',     'http://zhidao.baidu.com',     'http://news.baidu.com',     'http://xueshu.baidu.com'] def task(url):     response = requests.get(url)     print(response.text) if __name__ == '__main__':     p_list = []     start_time = time.time()     for url in url_lists:         p = Process(target=task, args=(url,))         p_list.append(p)         p.start()     for p in p_list:         p.join()     print("Runtime: {}".format(time.time() - start_time)) # Runtime: 1.91

　

3.进程池（1）

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34

import time import requests from concurrent.futures import ProcessPoolExecutor """ Py2里    没有线程池   但是有进程池 Py3里    有线程池     有进程池 """ url_lists = [     'http://www.baidu.com',     'http://fanyi.baidu.com',     'http://map.baidu.com',     'http://music.baidu.com/',     'http://tieba.baidu.com',     'http://v.baidu.com',     'http://image.baidu.com',     'http://zhidao.baidu.com',     'http://news.baidu.com',     'http://xueshu.baidu.com'] def task(url):     response = requests.get(url)     print(response.content) if __name__ == '__main__':     start_time = time.time()     pool = ProcessPoolExecutor(10)     for url in url_lists:         pool.submit(task,url)     pool.shutdown(wait=True)     print("Runtime: {}".format(time.time() - start_time)) # Runtime: 2.00

　　

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38

# 进程池 + 回调函数 import time import requests from concurrent.futures import ProcessPoolExecutor """ Py2里    没有线程池   但是有进程池 Py3里    有线程池     有进程池 """ url_lists = [     'http://www.baidu.com',     'http://fanyi.baidu.com',     'http://map.baidu.com',     'http://music.baidu.com/',     'http://tieba.baidu.com',     'http://v.baidu.com',     'http://image.baidu.com',     'http://zhidao.baidu.com',     'http://news.baidu.com',     'http://xueshu.baidu.com'] def task(url):     response = requests.get(url)     return response.content def callback(future):     print(future.result()) if __name__ == '__main__':     start_time = time.time()     pool = ProcessPoolExecutor(10)     for url in url_lists:         v = pool.submit(task,url)         v.add_done_callback(callback)     pool.shutdown(wait=True)     print("Runtime: {}".format(time.time() - start_time))

　　

3.进程池（2）

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33

import time import requests from  multiprocessing import Pool url_lists = [     'http://www.baidu.com',     'http://fanyi.baidu.com',     'http://map.baidu.com',     'http://music.baidu.com/',     'http://tieba.baidu.com',     'http://v.baidu.com',     'http://image.baidu.com',     'http://zhidao.baidu.com',     'http://news.baidu.com',     'http://xueshu.baidu.com'] def task(url):     response = requests.get(url)     return response.content def callBackFunc(content):     print(content) if __name__ == '__main__':     start_time = time.time()     pool = Pool(10)     for url in url_lists:         pool.apply_async(func=task,args=(url,),callback=callBackFunc)     pool.close()     pool.join()     print("Runtime: {}".format(time.time() - start_time)) # Runtime: 1.96

　

2019-03-06 補充

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21

# Parallel 底层調用的是 multiprocessing import time from joblib import Parallel, delayed def func(idx):     print(idx)     time.sleep(1)     return { 'idx':idx} start_ts = time.time() results = Parallel(-1)(     delayed(func)(x) for x in range(4) ) print(results) print('Runtime : {}'.format(time.time()-start_ts