python io密集型应用案例-Python IO密集型任务、计算密集型任务，以及多线程、多进程...

对于IO密集型任务：

直接执行用时：10.0333秒

多线程执行用时：4.0156秒

多进程执行用时：5.0182秒

说明多线程适合IO密集型任务。

对于计算密集型任务

直接执行用时：10.0273秒

多线程执行用时：13.247秒

多进程执行用时：6.8377秒

说明多进程适合计算密集型任务。

#coding=utf-8

import sys

import multiprocessing

import time

import threading

# 定义全局变量Queue

g_queue = multiprocessing.Queue()

def init_queue():

print("init g_queue start")

while not g_queue.empty():

g_queue.get()

for _index in range(10):

g_queue.put(_index)

print("init g_queue end")

return

# 定义一个IO密集型任务：利用time.sleep()

def task_io(task_id):

print("IOTask[%s] start" % task_id)

while not g_queue.empty():

time.sleep(1)

try:

data = g_queue.get(block=True, timeout=1)

print("IOTask[%s] get data: %s" % (task_id, data))

except Exception as excep:

print("IOTask[%s] error: %s" % (task_id, str(excep)))

print("IOTask[%s] end" % task_id)

return

g_search_list = list(range(10000))

# 定义一个计算密集型任务：利用一些复杂加减乘除、列表查找等

def task_cpu(task_id):

print("CPUTask[%s] start" % task_id)

while not g_queue.empty():

count = 0

for i in range(10000):

count += pow(3*2, 3*2) if i in g_search_list else 0

try:

data = g_queue.get(block=True, timeout=1)

print("CPUTask[%s] get data: %s" % (task_id, data))

except Exception as excep:

print("CPUTask[%s] error: %s" % (task_id, str(excep)))

print("CPUTask[%s] end" % task_id)

return task_id

if __name__ == "__main__":

print("cpu count:", multiprocessing.cpu_count(), " ")

print(u"========== 直接执行IO密集型任务 ==========")

init_queue()

time_0 = time.time()

task_io(0)

print(u"结束：", time.time() - time_0, " ")

print("========== 多线程执行IO密集型任务 ==========")

init_queue()

time_0 = time.time()

thread_list = [threading.Thread(target=task_io, args=(i,)) for i in range(10)]

for t in thread_list:

t.start()

for t in thread_list:

if t.is_alive():

t.join()

print("结束：", time.time() - time_0, " ")

print("========== 多进程执行IO密集型任务 ==========")

init_queue()

time_0 = time.time()

process_list = [multiprocessing.Process(target=task_io, args=(i,)) for i in range(multiprocessing.cpu_count())]

for p in process_list:

p.start()

for p in process_list:

if p.is_alive():

p.join()

print("结束：", time.time() - time_0, " ")

print("========== 直接执行CPU密集型任务 ==========")

init_queue()

time_0 = time.time()

task_cpu(0)

print("结束：", time.time() - time_0, " ")

print("========== 多线程执行CPU密集型任务 ==========")

init_queue()

time_0 = time.time()

thread_list = [threading.Thread(target=task_cpu, args=(i,)) for i in range(10)]

for t in thread_list:

t.start()

for t in thread_list:

if t.is_alive():

t.join()

print("结束：", time.time() - time_0, " ")

print("========== 多进程执行cpu密集型任务 ==========")

init_queue()

time_0 = time.time()

process_list = [multiprocessing.Process(target=task_cpu, args=(i,)) for i in range(multiprocessing.cpu_count())]

for p in process_list:

p.start()

for p in process_list:

if p.is_alive():

p.join()

print("结束：", time.time() - time_0, " ")

参考：https://zhuanlan.zhihu.com/p/24283040