线程池是一个保存了很多线程的结构,它主要的作用是线程的复用,减少线程创建和销毁所耗的时间以及节省系统资源。如果是单核cpu,使用多线程可能效率会低,因为线程切换回比较频繁,上下文保存和恢复会耗时,但是相对于进程来说开销还是小的。多核cpu的话,每个cpu都可以执行线程,这样就并发执行了,充分利用硬件。但是不管怎么说,创建过多的线程会带来上线文切换频繁,系统资源消耗。
由于最近考虑用python做服务端,自己实现了一个线程池,当做练手。可能还存在很多问题,如果大牛发现了,望指正。
#encoding:utf8
'''
Created on 2014年7月17日
@author: Leo.Liao
'''
import threading
import Queue
import time
import sys
#替我们工作的线程池中的线程
'''工作线程'''
class WorkThread(threading.Thread):
def __init__(self,name,workQueue):
threading.Thread.__init__(self, name=name)
self.workQueue = workQueue
# self.setDaemon(True)
'''设置一个flag信号,用来表示该线程是否还被dismiss,默认为false'''
self._dismissed = threading.Event()
self.start()
def run(self):
while True:
if self._dismissed.is_set():
return
try:
future = self.workQueue.get(block=False,timeout=None)
except:
continue
try:
'''执行函数'''
future.execute()
# time.sleep(0.2)
except:
print sys.exc_info()
'''关闭线程'''
def close(self):
self._dismissed.set()
'''线程池'''
class ThreadPool(object):
def __init__(self,minSize=10,maxSize=20):
# '''cpu个数'''
# self.__cpu_num = 1
# if 'NUMBER_OF_PROCESSORS' in os.environ:
# self.__cpu_num = os.environ['NUMBER_OF_PROCESSORS']
# if not minSize :
# minSize = self.__cpu_num*2
# if not maxSize :
# maxSize = self.__cpu_num*4
'''最小,最大线程数量'''
if minSize > maxSize :
self.__minSize = maxSize
self.__maxSize = minSize
else:
self.__minSize = minSize
self.__maxSize = maxSize
'''线程自增index'''
self.__index = 0
'''当前线程数'''
self.__currentSize = 0
'''线程的名称'''
self.__namePrefix = "thread-pooling-"
'''线程池'''
self.__pool = []
'''工作队列'''
self.__workQueue = Queue.Queue()
'''线程状态'''
self.__isClose = False
'''初始化线程'''
self.__initThreadPool()
'''关闭线程的检测频率'''
self.__closeInterval = 0.5
'''关闭线程池'''
self.__closeThread = threading.Thread( target = self.__monitorClose)
self.__closeThread.setDaemon(True)
self.__closeThread.start()
'''监控线程的频率'''
self.__sizeInterval = 2
'''监控线程池大小'''
self.__monitorSizeThread = threading.Thread(target = self.__monitorSize)
self.__monitorSizeThread.setDaemon(True)
self.__monitorSizeThread.start()
'''线程锁'''
self.__lock = threading.RLock()
'''线程池大小监控'''
def __monitorSize(self):
while True:
time.sleep( self.__sizeInterval)
if self.__workQueue.empty() and self.__isClose:
return
try:
self.__lock.acquire()
print "currentSize = %s, workQueue = %s, minSize = %s, maxSize = %s"%(self.__currentSize,self.__workQueue.qsize(),self.__minSize,self.__maxSize)
workQueueSize = self.__workQueue.qsize()
'''保持一定的线程数量'''
if workQueueSize < self.__currentSize and self.__currentSize > self.__minSize:
item = self.__pool.pop()
item.close()
self.__currentSize -= 1
print "close "
elif workQueueSize > self.__currentSize*2 and self.__currentSize < self.__maxSize:
thread = WorkThread(self.__namePrefix+str(self.__index),self.__workQueue)
self.__pool.append(thread)
self.__currentSize += 1
self.__index += 1;
print "create"
finally:
self.__lock.release()
'''监控线程池关闭'''
def __monitorClose(self):
while True:
time.sleep(self.__closeInterval)
if self.__workQueue.empty() and self.__isClose:
try:
self.__lock.acquire()
while self.__pool and len(self.__pool):
item = self.__pool.pop()
item.close()
return
finally:
self.__lock.release()
def __initThreadPool(self):
for i in range(0,self.__minSize):
self.__index = i
thread = WorkThread(self.__namePrefix+str(self.__index),self.__workQueue)
self.__pool.append(thread)
self.__currentSize = self.__minSize
'''添加工作'''
def executor(self,function, *args, **kwargs):
try:
assert self.__isClose == False,"threadpool has been closed."
poolLen = len(self.__pool)
if poolLen :
futrue = Future(function,*args,**kwargs)
self.__workQueue.put(futrue)
return futrue
except:
print sys.exc_info()
def close(self):
self.__isClose = True
# try:
# #等待所有线程完成。
# for item in self.pool:
#
# if item.isAlive():
# item.join()
# except:
# print sys.exc_info()
'''获取函数返回值'''
class Future(object):
def __init__(self,function,*args,**kwargs):
assert function,"function is none."
self.__function = function
self.__args = args
self.__kwargs = kwargs
self.__result = None
self.__condition = threading.Condition()
'''执行函数'''
def execute(self):
try:
self.__condition.acquire()
if self.__args and self.__kwargs:
self.__result = self.__function(*self.__args,**self.__kwargs)
elif self.__args and not self.__kwargs:
self.__result = self.__function(*self.__args)
elif self.__kwargs and not self.__args:
self.__result = self.__function(**self.__kwargs)
else:
self.__result = self.__function()
self.__condition.notify()
finally:
self.__condition.release()
'''获取结果'''
def get(self):
try:
self.__condition.acquire()
if not self.__result:
self.__condition.wait()
finally:
self.__condition.release()
return self.__result
def test_job( input , **kwargs):
html = threading.currentThread().name
try:
# print input
# print input[0]''
# print input[0][0]
# print kwargs
input.write(threading.currentThread().name+"\r\n")
# print "===================="
except:
print "happend"
print sys.exc_info()
# input[0][0].close()
return html
if __name__=="__main__":
try:
threadPool = ThreadPool(10)
input = open('data', 'w+')
resultQueue = []
for i in range(0,100000):
resultQueue.append(threadPool.executor(test_job,input))
while len(resultQueue):
futrue = resultQueue.pop()
print futrue.get()
threadPool.close()
except:
print "abc"
print sys.exc_info()