7.4实现一个多线程web视频监控服务器,需要对请求连接数做限制。以防止恶意用户发起大量连接而导致服务器创建大量线程,导致因资源耗尽瘫痪
可以使用线程池(提前创建好固定数量的线程,取还),替代原来的每次请求创建线程
# python3中有线程池实现
使用标准库中concurrent.futures下的ThreadPoolExecutor,对象的submit和map方法可以采用启动线程池中线程执行任务
from concurrent.futures import ThreadPoolExecutor
executor = ThreadPoolExecutor(3) # 指定线程池中有3个线程
def f(a,b):
print('f', a,b)
return a**b
future = executor.submit(f,2,3)
future.result() # 若计算复杂,会阻塞在这里
executor.map(f,[2,3,5],[4,5,6]) # 在多个线程上同时调用;2^4,3^5,5^6
若线程池中的无空闲线程,新任务将pending
import time
def f(a,b):
print('f',a,b)
time.sleep(10)
return a**b
executor.map(f,[2,3,5,6,7],[4,5,6,7,8])
# 7.4当前服务器使用的ThreadingTCPServer,现在要替换成线程池的解决,需要知道它的内部是如何创建线程的,找到相应的位置,改写成线程池的实现;需要在标准库中读源码
是一个多继承,ThreadingMinIn,TCPServer;线程相关是前一部分,ThreadingMinIn中找到线程创建相关,threading.Thread;我们就要继承ThreadingTCPServer,重写process_request()方法
#
import os,cv2,time,struct,threading
from BaseHTTPServer import HTTPServer,BaseHTTPRequestHandler
from SocketServer import TCPServer,ThreadingTCPServer
from threading import Thread,RLock
from select import select
class JpegStreamer(Thread):
def __init__(self, camera):
Thread.__init__(self)
self.cap = cv2.VideoCapture(camera)
self.lock = RLock()
self.pipes = {}
def register(self):# 数据的输出,获取源中的数据
pr, pw = os.pipe() # 创建管道
self.lock.acquire()
self.pipes[pr] = pw # 维护管道的写端
self.lock.release()
return pr # 管道的读端返回客户
def unregister(self, pr):
self.lock.acquire()
self.pipes.pop(pr)
self.lock.release()
pr.close()
pw.close()
def capture(self): # 数据采集部分
cap = self.cap
while cap.isOpened():
ret, frame = cap.read()
if ret:
#ret, data = cv2.imencode('.jpg', frame)
ret, data = cv2.imencode('.jpg', frame, (cv2.IMWRITE_JPEG_QUALITY, 40))# opencv的库,每次从摄像头读取1帧,并且编码成jpg图片返回
yield data.tostring() # 返回一个生成器对象
def send(self, frame): # 把1帧发送到所有注册的管道中去
n = struct.pack('l', len(frame))
self.lock.acquire()
if len(self.pipes):
_, pipes, _ = select([], self.pipes.itervalues(), [], 1)
for pipe in pipes:
os.write(pipe, n)
os.write(pipe, frame)
self.lock.release()
def run(self):
for frame in self.capture(): # 从capture中拿到1帧数据,发送所有管道
self.send(frame)
class JpegRetriever(object):
def __init__(self, streamer):
self.streamer = streamer
#self.pipe = streamer.register() # 每个线程使用的pipe实现成线程本地数据
self.local = threading.local() # 之后每注册一个pipe,都应该是local的属性
def __enter__(self): # 每次使用pipe需要注册,使用完后需要注销;使用上下文管理,把这个类实现成上下文管理器,实现__enter__方法
if hasattr(self.local, 'pipe'): # 为了避免重复进入enter,查看local对象是否包含pipe属性;
raise RuntimeError()
self.local.pipe = streamer.register() # 进行注册
return self.retrieve()
def retrieve(self): # 每次从管道中读取1帧数据,并返回
while True:
ns = os.read(self.local.pipe, 8)
n = struct.unpack('l', ns)[0]
data = os.read(self.local.pipe, n)
yield data
#def cleanup(self):
def __exit__(self, *args): # 不关注异常,*args收集
self.streamer.unregister(self.local.pipe)
del self.local.pipe # 使用完后,删除
return True # 压制异常
class Handler(BaseHTTTPRequestHandler):
retriever = None
@staticmethod
def setJpegRetriever(retriever):
Handler.retriever = retriever
def do_GET(self):
if self.retriever is None:
raise RuntimeError('no retriver')
if self.path != '/':
return
self.send_response(200)
self.send_header('Content-type', 'multipart/x-mixed-replace;boundary=abcde')
self.end_headers()
with self.retriever as frames: # 使用时,上下文管理
for frame in frames:
self.send_frame(frame)
def send_frame(self, frame):
self.wfile.write('--abcde\r\n')
self.wfile.write('Content-Type:image/jpeg\r\n')
self.wfile.write('Content-Length:%d\r\n\r\n' % len(frame))
self.wfile.write(frame)
from concurrent.futures import ThreadPoolExecutor
class ThreadingPoolTCPServer(ThreadingTCPServer):
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True,max_thread_num = 100): # ;ThreadingMinIn无构造器,就得关注另一个TCPServer;基础上,增加1个参数,指定线程数量
super().__init__(server_address, RequestHandlerClass, bind_and_activate=True)
self.executor = ThreadPoolExecutor(max_thread_num)
def process_request(self, request, client_address):
self.executor.submit(self.process_request_thread, client_address)
#t = threading.Thread(target = self.process_request_thread,args = (request, client_address))
#t.daemon = self.daemon_threads
#t.start()
if __name__ == '__main__':
streamer = JpegStreamer(0)
streamer.start()
retriever = JpegRetriever(streamer)
Handler.setJpegRetriever(retriever)
print 'Start server...'
httpd = ThreadingPoolTCPServer(('', 9000), Handler, max_thread_num = 100)
htttpd.serve_forever()
7.5【多线程与多进程】使用线程池
最新推荐文章于 2024-10-02 10:53:34 发布