python线程池技术

python线程池技术

标签（空格分隔）： python 线程池

---

整理自http://www.the5fire.com/python-thread-pool.html
http://blog.csdn.net/onwer3/article/details/9238807
https://www.ibm.com/developerworks/cn/aix/library/au-threadingpython/
https://segmentfault.com/a/1190000000414339
http://www.cnblogs.com/goodhacker/p/3359985.html

---

在此我们考虑
1、 使用线程的第三方库threadpool（类似与java中使用Executors.newCahedThreadPool)
2、 自行实现线程池

我们先考虑自己实现线程池

线程池的实现

在网上相信很容易就能找到python实现线程池的一个例子

# !/usr/bin/env python
# -*- coding:utf-8 -*-

import Queue
import threading
import time

class WorkManager(object):
    def __init__(self, work_num=1000,thread_num=2):
        self.work_queue = Queue.Queue()
        self.threads = []
        self.__init_work_queue(work_num)
        self.__init_thread_pool(thread_num)

    """
        初始化线程
    """
    def __init_thread_pool(self,thread_num):
        for i in range(thread_num):
            self.threads.append(Work(self.work_queue))

    """
        初始化工作队列
    """
    def __init_work_queue(self, jobs_num):
        for i in range(jobs_num):
            self.add_job(do_job, i)

    """
        添加一项工作入队
    """
    def add_job(self, func, *args):
        self.work_queue.put((func, list(args)))#任务入队，Queue内部实现了同步机制

    """
        等待所有线程运行完毕
    """
    def wait_allcomplete(self):
        for item in self.threads:
            if item.isAlive():item.join()


class Work(threading.Thread):
    def __init__(self, work_queue):
        threading.Thread.__init__(self)
        self.work_queue = work_queue
        self.start()

    def run(self):
        #死循环，从而让创建的线程在一定条件下关闭退出
        while True:
            try:
                do, args = self.work_queue.get(block=False)#任务异步出队，Queue内部实现了同步机制
                do(args)
                self.work_queue.task_done()#通知系统任务完成
            except:
                break

#具体要做的任务
def do_job(args):
    time.sleep(0.1)#模拟处理时间
    print threading.current_thread(), list(args)

if __name__ == '__main__':
    start = time.time()
    work_manager =  WorkManager(10000, 10)#或者work_manager =  WorkManager(10000, 20)
    work_manager.wait_allcomplete()
    end = time.time()
    print "cost all time: %s" % (end-start)

它的整个运行逻辑就是，给WorkManager分配制定的任务量和线程数，然后每个线程都从任务队列中获取任务来执行，直到队列中没有任务。这里面也用到了Queue内部的同步机制（至于是啥同步机制目前还没去研究）。

转自http://blog.randomk.org/posts/2014/01/python-threadpool/

import time
import random
import logging
import threading
import Queue

logging.basicConfig(level=logging.INFO, format='%(threadName)-8s : %(message)s')


class ThreadPoolException(Exception):

    """ Thread Pool Exception """
    pass


class WorkThread(threading.Thread):

    """ WorkThread """

    def __init__(self, work_queue, result_queue):
        super(WorkThread, self).__init__()
        self.__work_queue = work_queue
        self.__result_queue = result_queue

    def run(self):
        while True:
            content = self.__work_queue.get()
            if isinstance(content, str) and content == 'quit':
                break
            try:
                func, args, kwargs = content
                ret = func(*args, **kwargs)
            except:
                pass
            else:
                if ret:
                    self.__result_queue.put(ret)
            finally:
                self.__work_queue.task_done()


class ThreadPool(object):

    """ ThreadPoolManager """

    def __init__(self, thread_num=10):
        self.__thread_num = thread_num
        self.__work_queue = Queue.Queue()
        self.__result_queue = Queue.Queue()
        self.__start = False
        self.__finish = False

    def start(self):
        self.__start = True
        for _ in range(self.__thread_num):
            worker = WorkThread(self.__work_queue, self.__result_queue)
            worker.setDaemon(True)
            worker.start()

    def add(self, func, *args, **kwargs):
        self.__work_queue.put((func, args, kwargs))

    def join(self):
        if not self.__start:
            raise ThreadPoolException('Worker not started')
        self.__work_queue.join()
        self.__finish = True

    def close(self):
        if not self.__finish:
            raise ThreadPoolException('Worker not finished')
        for _ in xrange(self.__thread_num):
            self.__work_queue.put('quit')

    def get_result(self):
        if not self.__finish:
            raise ThreadPoolException('Worker not finished')
        results = []
        while not self.__result_queue.empty():
            ret = self.__result_queue.get()
            results.append(ret)
        return results


def task(i):
    """ Task """
    logging.info(i)
    time.sleep(2)
    return i


def main():
    """ Example """
    pool = ThreadPool(5)
    pool.start()

    for _ in range(20):
        pool.add(task, random.randint(1, 100))

    pool.join()
    pool.close()
    print pool.get_result()
if __name__ == '__main__':
    main()

总之就是大同小异

---

第三方threadpool

#!/usr/bin/env python
import threadpool
import time,random

def hello(str):
    time.sleep(2)
    return str

def print_result(request, result):
    print "the result is %s %r" % (request.requestID, result)

data = [random.randint(1,10) for i in range(20)]

pool = threadpool.ThreadPool(5)
requests = threadpool.makeRequests(hello, data, print_result)
[pool.putRequest(req) for req in requests]
pool.wait()