7.5【多线程与多进程】使用线程池

7.4实现一个多线程web视频监控服务器，需要对请求连接数做限制。以防止恶意用户发起大量连接而导致服务器创建大量线程，导致因资源耗尽瘫痪
可以使用线程池（提前创建好固定数量的线程，取还），替代原来的每次请求创建线程

# python3中有线程池实现
使用标准库中concurrent.futures下的ThreadPoolExecutor，对象的submit和map方法可以采用启动线程池中线程执行任务

from concurrent.futures import ThreadPoolExecutor
executor = ThreadPoolExecutor(3) # 指定线程池中有3个线程
def f(a,b):
	print('f', a,b)
	return a**b
future = executor.submit(f,2,3)
future.result() # 若计算复杂，会阻塞在这里
executor.map(f,[2,3,5],[4,5,6]) # 在多个线程上同时调用；2^4,3^5,5^6

若线程池中的无空闲线程，新任务将pending
import time
def f(a,b):
	print('f',a,b)
	time.sleep(10)
	return a**b
executor.map(f,[2,3,5,6,7],[4,5,6,7,8])

# 7.4当前服务器使用的ThreadingTCPServer，现在要替换成线程池的解决，需要知道它的内部是如何创建线程的，找到相应的位置，改写成线程池的实现；需要在标准库中读源码
是一个多继承，ThreadingMinIn,TCPServer；线程相关是前一部分，ThreadingMinIn中找到线程创建相关，threading.Thread；我们就要继承ThreadingTCPServer，重写process_request()方法

#
import os,cv2,time,struct,threading
from BaseHTTPServer import HTTPServer,BaseHTTPRequestHandler
from SocketServer import TCPServer,ThreadingTCPServer
from threading import Thread,RLock
from select import select

class JpegStreamer(Thread):
	def __init__(self, camera):
		Thread.__init__(self)
		self.cap = cv2.VideoCapture(camera)
		self.lock = RLock()
		self.pipes = {}
	def register(self):# 数据的输出，获取源中的数据
		pr, pw = os.pipe() # 创建管道
		self.lock.acquire()
		self.pipes[pr] = pw # 维护管道的写端
		self.lock.release()
		return pr # 管道的读端返回客户
	def unregister(self, pr):
		self.lock.acquire()
		self.pipes.pop(pr)
		self.lock.release()
		pr.close()
		pw.close()
	def capture(self): # 数据采集部分
		cap = self.cap
		while cap.isOpened():
			ret, frame = cap.read() 
			if ret:
				#ret, data = cv2.imencode('.jpg', frame)
				ret, data = cv2.imencode('.jpg', frame, (cv2.IMWRITE_JPEG_QUALITY, 40))# opencv的库，每次从摄像头读取1帧，并且编码成jpg图片返回
				yield data.tostring() # 返回一个生成器对象
	def send(self, frame): # 把1帧发送到所有注册的管道中去
		n = struct.pack('l', len(frame))
		self.lock.acquire()
		if len(self.pipes):
			_, pipes, _ = select([], self.pipes.itervalues(), [], 1)
			for pipe in pipes:
				os.write(pipe, n)
				os.write(pipe, frame)
		self.lock.release()
	def run(self):
		for frame in self.capture(): # 从capture中拿到1帧数据，发送所有管道
			self.send(frame)
				
class JpegRetriever(object):
	def __init__(self, streamer):
		self.streamer = streamer
		#self.pipe = streamer.register() # 每个线程使用的pipe实现成线程本地数据
		self.local = threading.local() # 之后每注册一个pipe，都应该是local的属性
	def __enter__(self): # 每次使用pipe需要注册，使用完后需要注销；使用上下文管理，把这个类实现成上下文管理器，实现__enter__方法
		if hasattr(self.local, 'pipe'): # 为了避免重复进入enter，查看local对象是否包含pipe属性；
			raise RuntimeError()
		self.local.pipe = streamer.register() # 进行注册
		return self.retrieve()
	def retrieve(self): # 每次从管道中读取1帧数据，并返回
		while True:
			ns = os.read(self.local.pipe, 8)
			n = struct.unpack('l', ns)[0]
			data = os.read(self.local.pipe, n)
			yield data
	#def cleanup(self):
	def __exit__(self, *args): # 不关注异常，*args收集
		self.streamer.unregister(self.local.pipe)
		del self.local.pipe # 使用完后，删除
		return True # 压制异常

class Handler(BaseHTTTPRequestHandler):
	retriever = None
	@staticmethod
	def setJpegRetriever(retriever):
		Handler.retriever = retriever
	def do_GET(self):
		if self.retriever is None:
			raise RuntimeError('no retriver')
		if self.path != '/':
			return
		self.send_response(200)
		self.send_header('Content-type', 'multipart/x-mixed-replace;boundary=abcde')
		self.end_headers()
		with self.retriever as frames: # 使用时，上下文管理
			for frame in frames:
				self.send_frame(frame)
	def send_frame(self, frame):
		self.wfile.write('--abcde\r\n')
		self.wfile.write('Content-Type:image/jpeg\r\n')
		self.wfile.write('Content-Length:%d\r\n\r\n' % len(frame))
		self.wfile.write(frame)

from concurrent.futures import ThreadPoolExecutor
class ThreadingPoolTCPServer(ThreadingTCPServer):
	def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True,max_thread_num = 100): # ；ThreadingMinIn无构造器，就得关注另一个TCPServer；基础上，增加1个参数，指定线程数量
		super().__init__(server_address, RequestHandlerClass, bind_and_activate=True)
		self.executor = ThreadPoolExecutor(max_thread_num)
	def process_request(self, request, client_address):
		self.executor.submit(self.process_request_thread, client_address)
		#t = threading.Thread(target = self.process_request_thread,args = (request, client_address))
		#t.daemon = self.daemon_threads
		#t.start()
		

if __name__ == '__main__':
	streamer = JpegStreamer(0)
	streamer.start()

	retriever = JpegRetriever(streamer)
	Handler.setJpegRetriever(retriever)

	print 'Start server...'
	httpd = ThreadingPoolTCPServer(('', 9000), Handler, max_thread_num = 100) 
	htttpd.serve_forever()