基础知识八、Python实现线程池之线程池的实现与测试

       通过前几节的铺垫本节我们就将实现一个完整的线程池（低配版）， 一个完整的线程池包括一个存放线程的队列（线程池）和存放待处理任务的队列，其中线程池中的线程通过消费任务队列中的任务对象来实现同步、异步调用的目的。
       线程池的处理流程大致如下：

一、线程池的实现

       包括start()和join()方法开启和停止当前线程池，put方法用来向任务队列添加任务对象。默认创建CPU核数两倍的线程处理对象。

class ThreadPool:

    def __init__(self, size=0):
        if not size:
            size = psutil.cpu_count() * 2
        print('size is %d' % size)
        self.size = size
        # 任务队列 存放待处理的任务
        self.task_queue = ThreadSafeQueue()
        # 线程池
        self.pool = ThreadSafeQueue(size)

        for i in range(self.size):
            print('put thread:%d' % i)
            self.pool.put(ThreadProcess(self.task_queue))

    def start(self):
        """
        开启线程池
        :return:
        """
        for i in range(self.pool.size()):
            print('start thread:%d' % i)
            thread = self.pool.get(i)
            thread.start()

    def join(self):
        """
        停止线程池
        :return:
        """
        for i in range(self.pool.size()):
            print('join thread:%d' % i)
            thread = self.pool.get(i)
            thread.stop()
        # 线程池未完全停止
        if not self.pool:
            thread = self.pool.pop()
            thread.join()

    def put(self, task):
        """
        将任务放入线程池
        :param task:
        :return:
        """
        if not isinstance(task, Task):
            raise TaskTypeException()
        res = self.task_queue.put(task)
        print('put task %d' % task.id)

        return res

    def batch_put(self, task_list):
        """
        向线程池中批量添加任务
        :param task_list:
        :return:
        """
        if not isinstance(task_list, list):
            task_list = list(task_list)

        return [self.put(task) for task in task_list]

    def size(self):
        """
        获取线程池的大小
        :return:
        """

        return self.pool.size()


class TaskTypeException(Exception):
    pass

二、测试逻辑

2.1 测试基本任务对象

       构造基本任务对象

class SimpleTask(Task):

    def __init__(self, func, *args, **kwargs):
        super(SimpleTask, self).__init__(func, *args, **kwargs)

2.1.1 测试添加单任务对象

def counter(name):
    sum = 0
    for i in range(100):
        sum += i
    print(name, sum)
    return sum


def task_test_1():
    start_time = time.time()

    thread_pool = ThreadPool()
    print('thread pool not start')
    thread_pool.start()
    print('thread pool already start')
    task = SimpleTask(func=counter, name=('thread'))
    thread_pool.put(task)

    end_time = time.time()
    print('total time is %d' % (end_time - start_time))

测试结果：

2.1.2 测试添加多个任务对象

def task_test_2():
    start_time = time.time()

    thread_pool = ThreadPool()
    print('thread pool not start')
    thread_pool.start()
    print('thread pool already start')
    for s in range(10):
        task = SimpleTask(func=counter, name=('thread' + str(s)))
        thread_pool.put(task)

    end_time = time.time()
    print('total time is %d' % (end_time - start_time))

测试结果：

2.2 测试异步任务对象

       构造异步任务对象

class SimpleAsyncTask(AsyncTask):

    def __init__(self, func, *args, **kwargs):
        super(SimpleAsyncTask, self).__init__(func, *args, **kwargs)

2.2.1 添加多个任务对象

def async_test_1():

    def async_counter(name):
        sum = 0
        for i in range(100):
            sum += i
        print(name, sum)
        return sum

    start_time = time.time()

    thread_pool = ThreadPool()
    print('thread pool not start')
    thread_pool.start()
    print('thread pool already start')
    for s in range(10):
        task = SimpleAsyncTask(func=async_counter, name=('task' + str(s)))
        thread_pool.put(task)
        print('result from thread %d is %s' % (task.id, task.get_result()))

    end_time = time.time()
    print('total time is %d' % (end_time - start_time))

测试结果：

2.2.2 测试等待任务执行结果

        先获取结果，阻塞当前线程并等待任务执行完毕

def async_test_2():

    def async_counter():
        sum = 0
        for i in range(100):
            sum += i
        time.sleep(5)
        return sum

    start_time = time.time()

    thread_pool = ThreadPool()
    print('thread pool not start')
    thread_pool.start()
    print('thread pool already start')
    task = SimpleAsyncTask(func=async_counter)
    thread_pool.put(task)
    print('result from thread %d is %s' % (task.id, task.get_result()))

    end_time = time.time()
    print('total time is %d' % (end_time - start_time))

测试结果:

2.2.3 测试稍后获取任务执行结果

       先sleep再获取结果

def async_test_3():

    def async_counter():
        sum = 0
        for i in range(100):
            sum += i
        return sum

    start_time = time.time()

    thread_pool = ThreadPool()
    print('thread pool not start')
    thread_pool.start()
    print('thread pool already start')
    task = SimpleAsyncTask(func=async_counter)
    thread_pool.put(task)
    time.sleep(5)
    print('result from thread %d is %s' % (task.id, task.get_result()))

    end_time = time.time()
    print('total time is %d' % (end_time - start_time))

测试结果：