python 多路复用示例

IO多路复用是处理高并发常用的一种处理手段，他可以有效的提高系统的吞吐率，处理的链接越多他体现的优势越明显。谈到这个名词，比较有代表性的是linux上的epoll、windows上的iocp、BSD上的kqueue，这些都是操作系统内核实现的功能，高级语言一般会对这些调用进行下封装，方便用户使用。这里主要是python上多路复用使用的几个例子，大同小异。python提供了select模块，在linux系统上可以使用 select.select、select.poll、select.epoll，在windows上只能使用select.select(官方没有提供iocp的支持)。为了方便用户使用，python还提供了seletors模块，可以直接使用selectors.DefaultSelector，他会帮你选择当前系统最优的多路复用方式。

1、select.select，可监听的套接字数量：linux 默认1024, windows 默认64，由于轮询机制，不宜设置过大

#!/usr/bin/env python
#coding: utf-8
import select
import socket
import queue
from time import sleep


server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
server.setblocking(False)

server_address = ('localhost', 9999)
print('starting up on {}:{}'.format(*server_address))
server.bind(server_address)

server.listen(10)

inputs = [server]

outputs = []

msgs = {}

while inputs:
    print('waiting for the next event')
    readable, writable, exceptional = select.select(inputs, outputs, inputs)

    for s in readable:
        if s is server:
            connection, client_address = s.accept()
            print('connection from', client_address)
            connection.setblocking(False)
            inputs.append(connection)
            msgs[connection] = queue.Queue()
        else:
            data = s.recv(1024)
            print(type(data), len(data))
            if data != b'':
                print('received {} from {}'.format(data, s.getpeername()))
                msgs[s].put(data)
                if s not in outputs:
                    outputs.append(s)
            else:
                print('closing {}', s.getpeername())
                if s in outputs:
                    outputs.remove(s)
                inputs.remove(s)
                s.close()
                del msgs[s]

    for s in writable:
        try:
            message_queue = msgs.get(s)
            send_data = ''
            if message_queue is not None:
                send_data = message_queue.get_nowait()
            else:
                print("has closed")
        except queue.Empty as e:
            print("queue of {} is empty".format(s))
            print(e)
            outputs.remove(s)
        else:
            if message_queue is not None:
                s.send(send_data)
            else:
                print('has closed')

    for s in exceptional:
        print('exception on {}'.format(s.getpeername()))
        inputs.remove(s)
        if s in outputs:
            outputs.remove(s)
        s.close()
        del msgs[s]

    sleep(1)

2、select.poll 无套接字数量限制，也是轮询机制

#!/usr/bin/env python
#coding: utf-8
import select
import socket
import queue
from time import sleep


server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
server.setblocking(False)

server_address = ('localhost', 9999)
print('starting up on {}:{}'.format(*server_address))
server.bind(server_address)

server.listen(10)

inputs = [server]

outputs = []
timeout = 1000

msgs = {}

READ_ONLY = select.POLLIN|select.POLLPRI|select.POLLHUP|select.POLLERR
READ_WRITE = select.POLLOUT|READ_ONLY

poller = select.poll()
poller.register(server, READ_ONLY)
fd_to_socket = {server.fileno():server}

while True:
    events = poller.poll(timeout)
    for fd, flag in events:
        s = fd_to_socket[fd]

        if flag & (select.POLLIN|select.POLLPRI):
            if s is server:
                conn, client_address = s.accept()
                print('connection {}'.format(client_address))
                conn.setblocking(False)
                fd_to_socket[conn.fileno()] = conn
                poller.register(conn, READ_ONLY)
                msgs[conn] = queue.Queue()
            else:
                data = s.recv(1024)
                if data:
                    print('received {} from {}'.format(data, s.getpeername()))
                    msgs[s].put(data)
                    poller.modify(s, READ_WRITE)
                else:
                    print('closing: {}'.format(s.getpeername()))
                    poller.unregister(s)
                    s.close()
                    del msgs[s]
        elif flag & select.POLLHUP:
            print('Closing {} HUP'.format(s.getpeername()))
            poller.unregister(s)
            s.close()
        elif flag & select.POLLOUT:
            try:
                next_msg = msgs[s].get_nowait()
            except queue.Empty:
                print('{} queue empty'.format(s.getpeername()))
                poller.modify(s, READ_ONLY)
            else:
                print('sending {} to {}'.format(next_msg, s.getpeername()))
                s.send(next_msg)
        elif flag & select.POLLERR:
            print('exception on {}'.format(s.getpeername()))
            poller.unregister(s)
            s.close()
            del msgs[s]

3、select.epoll 无套接字数量限制，事件触发回调机制，可设置水平触发和边缘触发

#!/usr/bin/env python
#coding: utf-8
import select
import socket
import queue
from time import sleep


server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
server.setblocking(False)

server_address = ('localhost', 9999)
print('starting up on {}:{}'.format(*server_address))
server.bind(server_address)

server.listen(10)

inputs = [server]

outputs = []
timeout = 1000

msgs = {}

READ_ONLY = select.EPOLLIN|select.EPOLLPRI|select.EPOLLERR|select.EPOLLET
READ_WRITE = select.EPOLLOUT|READ_ONLY

epoller = select.epoll()
epoller.register(server, READ_ONLY)
fd_to_socket = {server.fileno():server}

while True:
    events = epoller.poll(timeout)
    for fd, flag in events:
        s = fd_to_socket[fd]

        if flag & (select.EPOLLIN|select.EPOLLPRI):
            if s is server:
                conn, client_address = s.accept()
                print('connection {}'.format(client_address))
                conn.setblocking(False)
                fd_to_socket[conn.fileno()] = conn
                epoller.register(conn, READ_ONLY)
                msgs[conn] = queue.Queue()
            else:
                data = s.recv(1024)
                if data:
                    print('received {} from {}'.format(data, s.getpeername()))
                    msgs[s].put(data)
                    epoller.modify(s, READ_WRITE)
                else:
                    print('closing: {}'.format(s.getpeername()))
                    epoller.unregister(s)
                    s.close()
                    del msgs[s]
        elif flag & select.EPOLLHUP:
            print('关闭 {}'.format(s.getpeername()))
            poller.unregister(s)
            s.close()
        elif flag & select.EPOLLOUT:
            try:
                next_msg = msgs[s].get_nowait()
            except queue.Empty:
                print('{} queue empty'.format(s.getpeername()))
                epoller.modify(s, READ_ONLY)
            else:
                print('sending {} to {}'.format(next_msg, s.getpeername()))
                s.send(next_msg)
        elif flag & select.EPOLLERR:
            print('exception on {}'.format(s.getpeername()))
            epoller.unregister(s)
            s.close()
            del msgs[s]

4、selectors.DefaultSelector

#!/usr/bin/env python
#coding: utf-8
import selectors
import socket
import queue
from time import sleep


def accept(sock, mask):
    conn, addr = sock.accept()
    conn.setblocking(False)
    msgs[conn] = queue.Queue()
    selector.register(conn, selectors.EVENT_READ, read)


def read(conn, mask):
    data = conn.recv(1024)
    if data:
        print('recv {} from {}'.format(data, conn.getpeername()))
        msgs[conn].put(data)
        selector.modify(conn, selectors.EVENT_WRITE, write)
    else:
        del msgs[conn]
        selector.unregister(conn)
        conn.close()


def write(conn, mask):
    try:
        next_msg = msgs[conn].get_nowait()
    except queue.Empty:
        print('{} queue empty'.format(conn.getpeername()))
        selector.modify(conn, selectors.EVENT_READ, read)
    else:
        print('sending {} to {}'.format(next_msg, conn.getpeername()))
        conn.send(next_msg)


server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
server.setblocking(False)

server_address = ('localhost', 9999)
print('starting up on {}:{}'.format(*server_address))
server.bind(server_address)
server.listen(10)
msgs = {}

selector = selectors.DefaultSelector()
selector.register(server, selectors.EVENT_READ, accept)

while True:
    events = selector.select()
    for key, mask in events:
        callback = key.data
        callback(key.fileobj, mask)

下面这个是客户端示例：

import socket
import time


msgs = ('hello world ', 'how are you ', 'you are the best ')
server_addr = ('localhost', 9999)

sockets = [
    socket.socket(socket.AF_INET, socket.SOCK_STREAM),
    socket.socket(socket.AF_INET, socket.SOCK_STREAM),
    socket.socket(socket.AF_INET, socket.SOCK_STREAM),
    socket.socket(socket.AF_INET, socket.SOCK_STREAM),
]

for s in sockets:
    s.connect(server_addr)

for msg in msgs:
    for s in sockets:
        print('{}: sending {}'.format(s.getsockname(), msg))
        s.send(msg.encode('utf-8'))
        time.sleep(1)


for s in sockets:
    data = s.recv(1024)
    print('{}: received {}'.format(s.getsockname(), data))
    if data != '':
        print('closing socket {}'.format(s.getsockname()))
    s.close()

需要注意的问题：
我们在进行select的时候，如果被监听的文件描述符列表为空，在不同的操作系统上有不同的表现。在windows上会报异常，如下：

  File "C:\Users\Administrator\.pyenv\pyenv-win\versions\3.7.2\lib\selectors.py", line 323, in select
    r, w, _ = self._select(self._readers, self._writers, [], timeout)
  File "C:\Users\Administrator\.pyenv\pyenv-win\versions\3.7.2\lib\selectors.py", line 314, in _select
    r, w, x = select.select(r, w, w, timeout)
OSError: [WinError 10022] 提供了一个无效的参数。

在linux上不会报异常，只是被阻塞住。