《Python网络编程》Part2 使用多路复用套接字I/O提升性能
1、基于进程的程序，python的SocketServer模块提供了两个实用类：ForkingMixIn和ThreadingMixIn。
ForkingMixIn会为每个客户端请求派生出一个新的进程。而ThreadingMixIn会为客户端请求产生多个线程，共享资源，避免进程间通信的复杂操作。
先贴多进程程序：
ForkingMixIn.py:
import os
import socket
import threading
import SocketServer
SERVER_HOST = 'localhost'
SERVER_PORT = 0 #tells the kernel to pick up a port dynamically
BUF_SIZE = 1024
ECHO_MSG = 'Hello echo server!'
class ForkingClient():
"""A client to test forking server"""
def __init__(self,ip,port):
#create a socket
self.sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
#connect to the server
self.sock.connect((ip,port))
def run(self):
"""Client playing with the server"""
#send the data to server
current_process_id = os.getpid()
print 'PID %s Sending echo message to the server : "%s"' %(current_process_id,ECHO_MSG)
sent_data_length = self.sock.send(ECHO_MSG)
print "Sent: %d characters, so far..." %sent_data_length
#Display server response
response =self.sock.recv(BUF_SIZE)
print "PID %s received: %s " %(current_process_id, response[5:])
def shutdown(self):
"""Cleanup the client socket"""
self.sock.close()
class ForkingServerRequestHandler(SocketServer.BaseRequestHandler):
def handle(self):
#Send the echo back to the client
data = self.request.recv(BUF_SIZE)
current_process_id = os.getpid()
response = '%s: %s' %(current_process_id,data)
print "Server sending response[current_process_id: data] = [%s]" %response
self.request.send(response)
return
class ForkingServer(SocketServer.ForkingMixIn,
SocketServer.TCPServer,
):
"""Nothing to add here,inherited everything necessary from parents"""
pass
def main():
# Launch the server
server = ForkingServer((SERVER_HOST,SERVER_PORT),ForkingServerRequestHandler)
ip, port = server.server_address #Retrive the port number
server_thread = threading.Thread(target=server.serve_forever)
server_thread.setDaemon(True) #don't hang on exit
server_thread.start()
print 'Server loop runnibg PID: %s' %os.getpid()
#launch the client
client1 = ForkingClient(ip,port)
client1.run()
client2 = ForkingClient(ip,port)
client2.run()
#Clean them up
server.shutdown()
client1.shutdown()
client2.shutdown()
server.socket.close()
if __name__ == '__main__':
main()
运行效果感觉有点凌乱，和系统有关系？暂时没搞懂。
2、然后继续看多线程程序：
先创建一个服务器线程，并在后台启动。然后启动三个测试客户端，向服务器发送消息。作为响应，服务器吧消息回显给客户端。在服务器请求处理类的handle()方法中，我们取回了当前线程的信息并将其打印出来，这些信息在每次客户端链接中都不同。
在客户端和服务器的通信中用到了sendall()方法，以保证发送的数据无任何丢失。
ThreadingMixIn.py:
import os
import socket
import threading
import SocketServer
SERVER_HOST = 'localhost'
SERVER_PORT = 0 #tells the kernel to pick up a port dynamically
BUF_SIZE = 1024
#ECHO_MSG = 'Hello echo server!'
def client(ip,port,message):
"""A client to test threading mixin server """
#Connect to the server
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((ip,port))
try:
sock.sendall(message)
response = sock.recv(BUF_SIZE)
print "Client received: %s " %response
finally:
sock.close
#class ForkingClient():
"""A client to test forking server"""
def __init__(self,ip,port):
#create a socket
self.sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
#connect to the server
self.sock.connect((ip,port))
def run(self):
"""Client playing with the server"""
#send the data to server
current_process_id = os.getpid()
print 'PID %s Sending echo message to the server : "%s"' %(current_process_id,ECHO_MSG)
sent_data_length = self.sock.send(ECHO_MSG)
print "Sent: %d characters, so far..." %sent_data_length
#Display server response
response =self.sock.recv(BUF_SIZE)
print "PID %s received: %s " %(current_process_id, response[5:])
def shutdown(self):
"""Cleanup the client socket"""
self.sock.close()
class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
"""An example of threaded TCP request handle"""
def handle(self):
#Send the echo back to the client
data = self.request.recv(BUF_SIZE)
current_thread = threading.current_thread()
response = '%s: %s' %(current_thread.name,data)
self.request.sendall(response)
class ThreadTCPServer(SocketServer.ThreadingMixIn,SocketServer.TCPServer):
"""Nothing to add here,inherited everything necessary from parents"""
pass
if __name__ == '__main__':
# Run Server
server = ThreadTCPServer((SERVER_HOST,SERVER_PORT),ThreadedTCPRequestHandler)
ip,port = server.server_address # retrieve ip addresss
#Start a thread with the server --one thread per request
server_thread =threading.Thread(target=server.serve_forever)
#Exit the server thread when the main thread exits
server_thread.deamon =True
server_thread.start()
print "Server loop running on thread: %s " %server_thread.name
#Run client
client(ip,port,"Hello from client1")
client(ip,port,"Hello from client2")
client(ip,port,"Hello from client3")
#server cleanup
server.shutdown()
3、使用select.select编写一个聊天室服务器
在大型服务器应用程序中可能有几百个或几千个客户端同时连接服务器，此时为每个客户端创建单线程的线程或进程可能不大切实际。由于内存可用量受限，且主机的CPU能力有限，我们需要一种更好地技术来处理大量的客户端。Python提供了select模块能解决这一问题。
我们见编写一个搞笑的聊天室服务器，处理几百或更多数量的客户端链接。我们要使用select模块提供的select（）方法，让连天服务器和客户端所做的操作始终不会阻塞消息的接收发送。
该程序使用一个脚本就能启动客户端和服务器，执行脚本时要指定--name参数。只有在命令行中传入了--name=server，脚本才启动聊天室服务器。如果为--name参数指定了其他值，例如client1或client2，则脚本会启动聊天室客户端。聊天室服务器绑定的端口在命令行参数--port中指定。对大型应用程序而言，最好在不同的模块中编写服务器和客户端。
select_select.py:
import select
import socket
import sys
import signal
import cPickle
import struct
import argparse
SERVER_HOST = 'localhost'
CHAT_SERVER_NAME = 'server'
#Some utilities
def send(channel,*args):
buffer = cPickle.dumps(args)
value = socket.htonl(len(buffer))
size = struct.pack("L",value)
channel.send(size)
channel.send(buffer)
def recieve(channel):
size = struct.calcsize("L")
size = channel.recv(size)
try:
size =socket.ntonl(struct.unpack("L",size)[0])
except struct.error,e:
return ''
buf = ""
while len(buf) < size:
buf += channel.recv(size-len(buf))
return cPickle.loads(buf)[0]
class ChatServer(object):
"""An example chat server using select"""
def __init__(self,port,backlog=5):
self.client = 0
self.clientmap = {}
self.outputs = [] #list output sockets
self.server = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.server.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
self.server.bind((SERVER_HOST,port))
print 'Server listening to port :%s...'%port
self.server.listen(backlog)
#catch keyboard interrupts
signal.signal(signal.SIGINT,self.sighandler)
def sighandler(self,signum,frame):
"""Clean up client outputs """
#close the server
print 'shutting down server...'
#close existing client sockets
for output in self.outputs:
output.close()
self.server.close()
def get_client_name(self,client):
"""Return the name of the client"""
info =self.clientmap[client]
host,name = info[0][0],info[1]
return '@'.join((name,host))
##################################################################
def run(self):
inputs = [self.server,sys.stdin]
self.outputs = []
running = True
while running:
try:
readable,writeable,exceptional = select.select(inputs,self.outputs,[])
except select.error,e:
break
for sock in readable:
if sock ==self.server:
#handle the server socket
client ,address = self.server.accept()
print "Chat server: got connection %d from %s " %(client.fileno(),address)
#Read the login name
cname = receive(client).split('NAME: ')[1]
#compute client name and send back
self.clients += 1
send(client,'CLIENT: '+ str(address[0]))
inputs.append(client)
self.clientmap[client] =(address,cname)
#send joining information to other clients
msg = "\n(Connected: New client (%d) from %s" %(self.clients,self.get_client_name(client))
for output in self.outputs:
send(output,msg)
self.outputs.append(client)
elif sock == sys.stdin:
#handle standard input
junk = sys.stdin.readline()
running = False
else:
#handle all other sockets
try:
data = receive(sock)
if data:
#send as new client's message..
msg = '\n#[' + self.get_client_name(sock) + ']>>' + data
#send data to all except ourself
for output in self.outputs:
if output != sock:
send(output,msg)
else:
print "Chat server: %d hung up" % sock.fileno()
self.clients -=1
sock.close()
inputs.remove(sock)
self.outputs.remove(sock)
#Sending client leaving information to others
msg = '\n(Now hung up: Client from %s)' %self.get_client_name(sock)
for output in self.outputs:
send(output,msg)
except socket.error ,e:
#remove
inputs.remove(sock)
self.outputs.name(sock)
self.server.close()
class ChatClient(object):
"""A command line chat client using select"""
def __init__(self ,name , port ,host = SERVER_HOST):
self.name = name
self.connected = False
self.host = host
self.port = port
#Initial prompt
self.prompt = '['+'@'.join((name,socket.gethostname().split('.')[0])) + ']>'
try:
self.sock =socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.sock.connect((host,self.port))
print "Now connected to chat server@ port %d" %self.port
self.connected = True
#send by name ...
send(self.sock,'NAME: ' + self.name)
data = receive(self.sock)
#contains client address,set it
addr = data.split('CLIENT: ')[1]
self.prompt = '[' + '@'.join((self.name,addr)) + ']> '
except socket.error ,e:
print "Failed to connect to chat server @ port %d " %self.port
sys.exit(1)
def run(self):
"""Chat client main loop"""
while self.connected:
try:
sys.stdout.write(self.prompt)
sys.stdout.flush()
#Wait for input from stdin and socket
readable,writeable,exceptional = select.select([0,self.sock],[],[])
for sock in readable:
if sock == 0:
data = sys.stdin.readline().strip()
if data: send(self.sock,data)
elif sock == self.sock:
data = receive(self.sock)
if not data:
print 'Client shutting down.'
self.connected =False
break
else :
sys.stdout.write(data + '\n')
sys.stdout.flush()
except KeyboardInterrupt:
print " Client interrupted."
self.sock.close()
break
if __name__ == "__main__" :
parser = argparse.ArgumentParser(description='Sock Server Example with Select')
parser.add_argument('--name',action = "store",dest = "name",required=True)
parser.add_argument('--port',action = "store",dest = "port",type = int ,required=True)
given_args = parser.parse_args()
port = given_args.port
name = given_args.name
if name == CHAT_SERVER_NAME:
server = ChatServer(port)
server.run()
else:
client = ChatClient(name=name,port = port)
client.run()
独立运行三次。
分别在三个窗口中输入:
select_select.py --name=server --port=8800
select_select.py --name=client1 --port=8800
select_select.py --name=client2 --port==8800
运行后服务器端可以连上，但是客户端链接失败。
4、select.epoll多路复用Web服务器
Python的select模块中有很多针对特定平台的网络时间管理函数。在Linux设备中可以使用epoll。这个函数利用操作系统内核轮询网络时间，让脚本知道有时间发生了。听起来比前一个例子中的select.select方案更高效。
我们来编写一个简单的web服务器，向每一个链接服务器的网页浏览器返回一行文本。
这个脚本的核心在web服务器的初始化过程中，我们要调用方法select.epoll（），注册服务器的文件描述符，以达到事件通知的目的。在web服务器执行的代码中，套接字事件由下述代码监控。
在Web服务器EpollServer的构造方法中创建了一个套接字服务器，绑定到本地主机指定端口上。服务器的套接字被设定为非阻塞模式（setblocking（0）），并设定了TCP_ NODELAY选项，让服务器无需缓冲便可直接交换数据（比如在SSH连接中）。然后创建一个select.epoll（）实例，再把套接字的文件描述符传给这个实例，以便监控。
在这个Web服务器的run（）方法中开始监听套接字事件。事件由下述常量表示：
#EPOLLIN:套接字读事件
#EPOLLOUT:套接字写事件
这个套接字服务器响应设为SERVER_RESPONSE。如果连接套接字服务器的客户端想写数据，可以在EPOLLOUT事件中完成。发生内部错误时，EPOLLHUP事件会把一个异常关闭信号发给套接字服务器。
import socket
import select
import argparse
SERVER_HOST = 'localhost'
EOL1 = b'\n\n'
EOL2 = b'\n\r\n'
SERVER_RESPONSE = b"""Http/1.1 200 OK\r\nDate: Mon,1 Apr 2013 01:01:01
GMT\r\nContent-Type: text/plain\r\nContent-Length: 25\r\n\r\n
Hello from Epoll server"""
class EpollServer(object):
"""A socket server using Epoll"""
def __init__(self,host=SERVER_HOST,port=0):
self.sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
self.sock.bind((host,port))
self.sock.listen(1)
self.sock.setblocking(0)
self.sock.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
print "Started Epoll Server"
self.epoll = select.epoll()
self.epoll.register(self.sock.fileno(),select.EPOLLIN)
def run(self):
"""Excutes epoll server operation"""
try:
connections ={};request = {}; response ={}
while True:
events = self.epoll.poll(1)
for fileno ,event in events:
if fileno == self.sock.fileno():
connection,address = self.sock.accept()
connection.setblocking(0)
self.epoll.register(connection.fileno(),select.EPOLLIN)
connections[connection.fileno()] = connection
requests[connection.fileno()] = b''
responses[connection.fileno()] = SERVER_RESPONSE
elif event&select.EPOLLOUT:
byteswritten = connections[fileno].send(responses[fileno])
responses[fileno] = responses[fileno][bytewritten:]
if len(responses[fileno]) == 0:
self.epoll.unregister(fileno)
connections[fileno].close()
del connections[fileno]
finally:
self.epoll.unregister(self.sock.fileno())
self.epoll.close()
self.sock.close()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description = 'Sock Server Example with Epoll')
parser.add_argument('--port',action="store",dest = "port",type=int,required=True)
given_args = parser.parse_args()
port = given_args.port
server = EpollServer(host=SERVER_HOST,port=port)
server.run()
5、使用并发库Diesel多路复用回显服务器
如果想编写一个大型自定义的网络应用程序，但不想重复输入初始化服务器的代码，比如说创建套接字、绑定地址、监听以及处理基本的错误等。有很多python网络库都可以帮助你把样板代码删除。这里我们要使用一个提供这种功能的库，叫做diesel。
Diesel使用非阻塞和协程技术提升编写网络服务器的效率。Diesel的网站上有这么一句话：“Diesel的核心是一个紧密的事件轮询，使用epoll提供几近平稳的性能，即便有10 000个或更多的连接也无妨。” 这一节我们通过一个简单的回显服务器介绍Diesel的用法。
首先需要安装Diesel3.0或者更新的版本，使用pip命令即可完成pip install diesel >=3.0
在python的diesel框架中，应用程序使用application（）类的实例初始化，时间处理函数注册在这个实例上。
import diesel
import argparse
class EchoServer(object):
"""An echo server using diesel"""
def handler(self,ermote_addr):
"""Runs the echo server"""
host, port = remote_addr[0],remote_addr[1]
print "Echo client connected from : %s: %d" %(host,port)
while True:
try:
message = diesel.until_eol()
your_message = ': '.join(['You said',message])
diesel.send(your_message)
except Exception,e:
print "Exception: ",e
def main(server_port):
app = diesel.Application()
server = EchoServer()
app.add_service(diesel.Service(server.handler,server_port))
app.run()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Echo server example with Diesel')
parser.add_argument('--port',action="store",dest="port",type=int,required=True)
given_args = parser.parse_args()
port = given_args.port
main(port)
1)、运行脚本echo_server_with_diesel.py --port=8800作为服务器端
2)、在另一个终端窗口中可以使用Telnet客户端连接回显服务器，测试消息回显：telnet localhost 8800，运行后即可测试消息回显。
3）原理：脚本从命令行参数--port中获取端口号，将其传给main（）函数。Diesel应用程序在main（）函数中初始化并运行。
在Diesel中有“服务”的概念，应用程序可以提供多种服务。EchoServer类中定义了handler（）方法，让服务器能够处理单独的客户端链接。运行服务时要把handler（）方法和客户端口号作为参数传给service（）方法。
handler（）方法决定服务器的行为，在这个脚本中，服务器直接返回消息文本。
如果把这个脚本和上节的对比，发现这里不需要编写样板代码，因此很容易吧精力集中在高层应用逻辑上。
扫一扫
276
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
