文章目录
通过前几节的铺垫本节我们就将实现一个完整的线程池(低配版), 一个完整的线程池包括一个存放线程的队列(线程池)和存放待处理任务的队列,其中线程池中的线程通过消费任务队列中的任务对象来实现同步、异步调用的目的。
线程池的处理流程大致如下:
一、线程池的实现
包括start()和join()方法开启和停止当前线程池,put方法用来向任务队列添加任务对象。默认创建CPU核数两倍的线程处理对象。
class ThreadPool:
def __init__(self, size=0):
if not size:
size = psutil.cpu_count() * 2
print('size is %d' % size)
self.size = size
# 任务队列 存放待处理的任务
self.task_queue = ThreadSafeQueue()
# 线程池
self.pool = ThreadSafeQueue(size)
for i in range(self.size):
print('put thread:%d' % i)
self.pool.put(ThreadProcess(self.task_queue))
def start(self):
"""
开启线程池
:return:
"""
for i in range(self.pool.size()):
print('start thread:%d' % i)
thread = self.pool.get(i)
thread.start()
def join(self):
"""
停止线程池
:return:
"""
for i in range(self.pool.size()):
print('join thread:%d' % i)
thread = self.pool.get(i)
thread.stop()
# 线程池未完全停止
if not self.pool:
thread = self.pool.pop()
thread.join()
def put(self, task):
"""
将任务放入线程池
:param task:
:return:
"""
if not isinstance(task, Task):
raise TaskTypeException()
res = self.task_queue.put(task)
print('put task %d' % task.id)
return res
def batch_put(self, task_list):
"""
向线程池中批量添加任务
:param task_list:
:return:
"""
if not isinstance(task_list, list):
task_list = list(task_list)
return [self.put(task) for task in task_list]
def size(self):
"""
获取线程池的大小
:return:
"""
return self.pool.size()
class TaskTypeException(Exception):
pass
二、测试逻辑
2.1 测试基本任务对象
构造基本任务对象
class SimpleTask(Task):
def __init__(self, func, *args, **kwargs):
super(SimpleTask, self).__init__(func, *args, **kwargs)
2.1.1 测试添加单任务对象
def counter(name):
sum = 0
for i in range(100):
sum += i
print(name, sum)
return sum
def task_test_1():
start_time = time.time()
thread_pool = ThreadPool()
print('thread pool not start')
thread_pool.start()
print('thread pool already start')
task = SimpleTask(func=counter, name=('thread'))
thread_pool.put(task)
end_time = time.time()
print('total time is %d' % (end_time - start_time))
测试结果:
2.1.2 测试添加多个任务对象
def task_test_2():
start_time = time.time()
thread_pool = ThreadPool()
print('thread pool not start')
thread_pool.start()
print('thread pool already start')
for s in range(10):
task = SimpleTask(func=counter, name=('thread' + str(s)))
thread_pool.put(task)
end_time = time.time()
print('total time is %d' % (end_time - start_time))
测试结果:
2.2 测试异步任务对象
构造异步任务对象
class SimpleAsyncTask(AsyncTask):
def __init__(self, func, *args, **kwargs):
super(SimpleAsyncTask, self).__init__(func, *args, **kwargs)
2.2.1 添加多个任务对象
def async_test_1():
def async_counter(name):
sum = 0
for i in range(100):
sum += i
print(name, sum)
return sum
start_time = time.time()
thread_pool = ThreadPool()
print('thread pool not start')
thread_pool.start()
print('thread pool already start')
for s in range(10):
task = SimpleAsyncTask(func=async_counter, name=('task' + str(s)))
thread_pool.put(task)
print('result from thread %d is %s' % (task.id, task.get_result()))
end_time = time.time()
print('total time is %d' % (end_time - start_time))
测试结果:
2.2.2 测试等待任务执行结果
先获取结果,阻塞当前线程并等待任务执行完毕
def async_test_2():
def async_counter():
sum = 0
for i in range(100):
sum += i
time.sleep(5)
return sum
start_time = time.time()
thread_pool = ThreadPool()
print('thread pool not start')
thread_pool.start()
print('thread pool already start')
task = SimpleAsyncTask(func=async_counter)
thread_pool.put(task)
print('result from thread %d is %s' % (task.id, task.get_result()))
end_time = time.time()
print('total time is %d' % (end_time - start_time))
测试结果:
2.2.3 测试稍后获取任务执行结果
先sleep再获取结果
def async_test_3():
def async_counter():
sum = 0
for i in range(100):
sum += i
return sum
start_time = time.time()
thread_pool = ThreadPool()
print('thread pool not start')
thread_pool.start()
print('thread pool already start')
task = SimpleAsyncTask(func=async_counter)
thread_pool.put(task)
time.sleep(5)
print('result from thread %d is %s' % (task.id, task.get_result()))
end_time = time.time()
print('total time is %d' % (end_time - start_time))
测试结果: