python线程池_python线程池实现

#!/bin/env python#-*- coding:utf-8 -*-

"""@lx

created on 2016-04-14"""

importQueueimportsysimportthreadingimporttimeimportStringIOimporttraceback

reload(sys)

sys.setdefaultencoding("utf8")classMyThread(threading.Thread):"""Background thread connected to the requests/results queues."""

def __init__(self, workQueue, resultQueue, timeout=0.1, **kwds):

threading.Thread.__init__(self, **kwds)

self.setDaemon(True)

self._workQueue=workQueue

self._resultQueue=resultQueue

self._timeout=timeout

self._dismissed=threading.Event()

self.start()defrun(self):"""Repeatedly process the job queue until told to exit."""

whileTrue:ifself._dismissed.isSet():breakhandlerKey= None #unique key

code = 0 #callback return code

handlerRet =None

errMsg= ""

try:

callable, args, kwds=self._workQueue.get(True, self._timeout)exceptQueue.Empty:continue

except:

exceptMsg=StringIO.StringIO()

traceback.print_exc(file=exceptMsg)

errMsg=exceptMsg.getvalue()

code= 3301 #system error

self._resultQueue.put(

(handlerKey, code, (callable, args, kwds), errMsg))break

ifself._dismissed.isSet():

self._workQueue.put((callable, args, kwds))break

try:if "handlerKey" inkwds:

handlerKey= kwds["handlerKey"]

handlerRet= callable(*args, **kwds) #block

self._resultQueue.put((handlerKey, code, handlerRet, errMsg))except:

exceptMsg=StringIO.StringIO()

traceback.print_exc(file=exceptMsg)

errMsg=exceptMsg.getvalue()

code= 3303self._resultQueue.put((handlerKey, code, handlerRet, errMsg))defdismiss(self):"""Sets a flag to tell the thread to exit when done with current job."""self._dismissed.set()classThreadPool(object):def __init__(self, workerNums=3, timeout=0.1):

self._workerNums=workerNums

self._timeout=timeout

self._workQueue= Queue.Queue() #no maximum

self._resultQueue =Queue.Queue()

self.workers=[]

self.dismissedWorkers=[]

self._createWorkers(self._workerNums)def_createWorkers(self, workerNums):"""Add num_workers worker threads to the pool."""

for i inrange(workerNums):

worker=MyThread(self._workQueue, self._resultQueue,

timeout=self._timeout)

self.workers.append(worker)def _dismissWorkers(self, workerNums, _join=False):"""Tell num_workers worker threads to quit after their current task."""dismissList=[]for i inrange(min(workerNums, len(self.workers))):

worker=self.workers.pop()

worker.dismiss()

dismissList.append(worker)if_join:for worker indismissList:

worker.join()else:

self.dismissedWorkers.extend(dismissList)def_joinAllDissmissedWorkers(self):"""Perform Thread.join() on all

worker threads that have been dismissed."""

for worker inself.dismissedWorkers:

worker.join()

self.dismissedWorkers=[]def addJob(self, callable, *args, **kwds):

self._workQueue.put((callable, args, kwds))def getResult(self, block=False, timeout=0.1):try:

item=self._resultQueue.get(block, timeout)returnitemexceptQueue.Empty, e:returnNoneexcept:raise

def waitForComplete(self, timeout=0.1):"""Last function. To dismiss all worker threads. Delete ThreadPool.

:param timeout"""

whileTrue:

workerNums= self._workQueue.qsize() #释放掉所有线程

runWorkers =len(self.workers)if 0 ==workerNums:

time.sleep(timeout)#waiting for thread to do job

self._dismissWorkers(runWorkers)break

#if workerNums < runWorkers: # 不能这样子乱取消

#self._dismissWorkers(runWorkers - workerNums)

time.sleep(timeout)

self._joinAllDissmissedWorkers()if "__main__" == __name__:

test1= """def doSomething(*args, **kwds):

if "sleep" in kwds:

sleep = kwds["sleep"]

msgTxt = "sleep %fs.." % sleep

time.sleep(sleep)

return msgTxt

for i in range(10):

print doSomething(sleep=0.1, handlerKey="key-%d"%i)

wm = ThreadPool(10)

for i in range(10):

wm.addJob(doSomething, sleep=1, handlerKey="key-%d"%i)

wm.waitForComplete()

for i in range(10):

print wm.getResult()

del wm"""

#test2 = """

def doSomething_(*args, **kwds):

sleep=int(args[0])

msgTxt= "sleep %ds.." %sleep

time.sleep(sleep)returnmsgTxt

wm= ThreadPool(10)

result=[]for i in range(10):

data= 5wm.addJob(doSomething_, data)while 1:

res=wm.getResult()ifres:

result.append(res)if 10 ==len(result):break

print "sleep 0.1"time.sleep(0.1)printtime.time()

wm.waitForComplete()printtime.time()#"""