Python3 socket与并发并行

最新推荐文章于 2021-09-27 11:39:57 发布

chizhi6248

最新推荐文章于 2021-09-27 11:39:57 发布

阅读量1k

点赞数

文章标签： python

原文链接：https://my.oschina.net/charlock/blog/861483

版权

1.socket概述

server_socket = socket.socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) # 创建socket
# AF指定协议类型,有AF_UNIX AF_INET AF_INET6
# SOCK指定套接字类型,有SOCK_STREAM SOCK_DGRAM SOCK_RAW SOCK_RDM SEQPACKET
server_socket.setsockopt(level, optname, value) # 设置socket的参数
server_socket.bind(address)
# 绑定IP 端口
server_socket.listen([backlog])
# 启用socket以接受连接,如果指定了backlog,它必须至少为0,它指定拒绝新连接之前系统将允许的未接受连接的数量,如果未指定，则选择默认的合理值
socket.accept()
# 接受连接,返回(conn, address),返回值是一对（conn，address）,(客户端)其中conn是可用于在连接上发送和接收数据的新套接字对象client_socket,而address是绑定到连接另一端的套接字的地址

client_socket.recv(bufsize[, flags])
# 从套接字接收客户端数据,返回值是接收的数据的字节对象.bufsize指定一次性接收的最大数据量
client_socket.send(bytes[, flags])
# 将数据发送给远程client_socket
# 大概知道socket之后呢,来看一个简单的server与client交互的例子

2.流程

# server.py建立一个server端的socket
import socket

HOST = ''                 # Symbolic name meaning all available interfaces
PORT = 50007              # Arbitrary non-privileged port
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: # with快新建socket
    s.bind((HOST, PORT)) # socket绑定地址，监听
    s.listen(1) # 开启socket连接，最大未连接数1
    conn, addr = s.accept() # 接收客户端连接
    with conn: # 对连接进行处理
        print('Connected by', addr)
        while True:
            data = conn.recv(1024) # 接收连接的数据
            if not data: break
            conn.sendall(data) # 原样发送回客户端

# client.py建立一个client端的socket
import socket

HOST = ''                 # The remote host
PORT = 50007              # The same port as used by the server
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
    s.connect((HOST, PORT)) # 连接到服务端
    s.sendall(b'Hello, world') # 发送数据给服务端
    data = s.recv(1024) # 接收服务端发送的数据
print('Received', repr(data))

$python server.py # 运行服务端等待连接
Connected by ('127.0.0.1', 35518)
$python client.py # 运行客户端连接
Received b'Hello, world'
# 接下来看看socket与并发

3.用socket处理斐波那契

# fib.py
def fib(n):
	if n <= 2:
		return 1
	else:
		return fib(n-1) + fib(n-2)

编写一个斐波那契数列的递归函数，客户端发送n，服务端返回对应的fib

# server.py
from socket import *
from fib import fib
def fib_server(address):
    sock = socket(AF_INET, SOCK_STREAM)
    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    sock.bind(address)
    sock.listen(5)
    while True:
        client, addr = sock.accept()
        print("Connection", addr)
        fib_handler(client)

def fib_handler(client):
    while True:
        req = client.recv(100)
        if not req:
            break
        n = int(req)
        result = fib(n)
        resp = str(result).encode('ascii') + b'\n'
        client.send(resp)
    print("Closed")

fib_server(('', 25000))

编写server，以socket模式运行

# 终端1，运行server
$python3 server.py

# 终端2，作为client1发起连接
$nc localhost 25000
1
1
10
55

# 终端3，作为client2发起连接
$nc localhost 25000
# 会堵塞，因为终端2已经连接server，那么如何让server并发的处理多个client呢

4.并发的socket

# server.py
from threading import Thread
def fib_server(address):
    sock = socket(AF_INET, SOCK_STREAM)
    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    sock.bind(address)
    sock.listen(5)
    while True:
        client, addr = sock.accept()
        print("Connection", addr)
        #fib_handler(client)
        Thread(target=fib_handler, args=(client, ), daemon=True).start()

设置守护线程运行

# 终端0，运行server
$python3 server.py 
Connection ('127.0.0.1', 45564)
Connection ('127.0.0.1', 45566)

# 终端1，运行client1
$nc localhost 25000
2
1
10
55

# 终端2，运行client2
$nc localhost 25000
1
1
10
55
# 那server对各个client的负载是均衡的么

5.并发下的线程负载

# perf2.py
from socket import *
import time

sock = socket(AF_INET, SOCK_STREAM)
sock.connect(('localhost', 25000))

n = 0
# 设计一个监控函数，1秒停顿，在时间片上清空次数n
from threading import Thread
def monitor():
    global n
    while True:
        time.sleep(1)
        print(n, 'reqs/sec')
        n = 0
Thread(target=monitor).start()
# 设计一个客户端，死循环的发送数据，用n记录连接次数
while True:
    sock.send(b'1')
    reps = sock.recv(100)
    n += 1
# 可以这么理解，在1秒停顿内连接次数

# 终端3,运行client3
$python perf2.py
24208 reqs/sec
23884 reqs/sec
24000 reqs/sec
22907 reqs/sec
22907 reqs/sec
22969 reqs/sec
# 这时候,我们在终端2输入40,client2发送40给server,请求其计算40的fib
$nc localhost 25000
40
# 这时候,server的进程会在计算40的fib,再看看终端3
4764 reqs/sec
4743 reqs/sec
4875 reqs/sec
4688 reqs/sec
# 由于server端在处理client2的请求(被占用),client3的recv就会延迟.在monitor停顿的1秒内连接明显减少

# CPU占用?
# 等待client2得到返回值，或者重启server(关闭client2是没用的，server会处理完client2的请求)
# client3恢复原来的状况
22711 reqs/sec
22599 reqs/sec
23534 reqs/sec
# 在终端4上,直接运行fib,让其计算50的fib
python -i fib.py 
>>> fib(1)
1
>>> fib(50)

# 在终端3上,client3连接次数减少,但较之前并不明显
15244 reqs/sec
15181 reqs/sec
15327 reqs/sec
# 所以,client2 client3消耗server进程被分配的资源,并不会再去CPU申请更多资源来满足
# server在处理client2和client3时,并不能独立平等的对待,client3会被client2影响.那么如何让两个客户端并行的执行呢

6.进程池下的并行

# server.py
from concurrent.futures import ProcessPoolExecutor as Pool

pool = Pool(4)

def fib_handler(client):
    while True:
        req = client.recv(100)
        if not req:
            break
        n = int(req)
        future = pool.submit(fib, n) # 把要执行fib和参数n提交到进程池
        result = future.result() # 取出进程池运行完的结果
        resp = str(result).encode('ascii') + b'\n'
        client.send(resp)
    print("Closed")

# 重启server, client2, client3,再测试
1966 reqs/sec
1967 reqs/sec
1835 reqs/sec
# 终端3,相比之前,client3的链接次数降低
# 在client2输入40
1970 reqs/sec
1993 reqs/sec
1864 reqs/sec
1926 reqs/sec
# client3不受client2的影响
# 为什么会这样呢

7.deque select

# 再去探究为什么前，简述下deque, selelct
class collections.deque([iterable[, maxlen]])
# 返回一个从左到右初始化的deque对象（使用append（））和iterable中的数据.如果未指定iterable,则新deque为空.
# Deques是栈和队列的泛化. Deques支持线程安全,内存高效的附加和从队列的任一边的弹出,在任一方向上具有大致相同的O（1）性能.
select.select(rlist, wlist, xlist[, timeout])
# 接受等待读取,等待写入,等待异常

8.生成器控制程序块执行

from socket import *
from fib import fib
from collections import deque
from select import select
from concurrent.futures import ThreadPoolExecutor as Pool

pool = Pool(4)

tasks = deque()
recv_wait = { }         # Mapping sockets -> tasks (generators)
send_wait = { }
def run():
    while any([tasks, recv_wait, send_wait]):
# 在task
        while not tasks:
            # 9.没有块程序可以执行了
            # No active tasks to run
            # wait for I/O
            can_recv, can_send, _ = select(recv_wait, send_wait, [])
            # 10.挑出正在读写的块程序,它们还没执行完
            for s in can_recv:
                tasks.append(recv_wait.pop(s))
            for s in can_send:
                tasks.append(send_wait.pop(s))
            # 11.把没执行完的程序,又压到tasks作为新一批块程序
        # 5.在tasks里有要执行的块程序时,执行完这一批的块程序
        task = tasks.popleft()
        # 6.从左边取出块程序,
        try:
            why, what = next(task)      # Run to the yield
            # 7.执行完这一块
            if why == 'recv':
                # Must go wait somewhere
                recv_wait[what] = task
            elif why == 'send':
                send_wait[what] = task
            else:
                raise RuntimeError("ARG!")
            # 8.把块分类,执行到recv的一类,执行到send的一类
        except StopIteration:
            print("task done")

def fib_server(address):
    sock = socket(AF_INET, SOCK_STREAM)
    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    sock.bind(address)
    sock.listen(5)
    while True:
        yield 'recv', sock # 3.一开始只有一个,
        client, addr = sock.accept() # blocking
        print("Connection", addr)
        tasks.append(fib_handler(client))

def fib_handler(client):
    while True:
        yield 'recv', client # 4.这里yield的时候,因为暂停,可以accept多个客户端连接
        req = client.recv(100) # blocking
        if not req:
            break
        n = int(req)
        future = pool.submit(fib, n)  # 把要执行fib和参数n提交到线程池
        result = future.result()
        resp = str(result).encode('ascii') + b'\n'
        yield 'send', client
        client.send(resp)   # blocking
    print("Closed")
# 1.在server运行从程序里,yield来暂停分隔开三个块程序
tasks.append(fib_server(('', 25000)))
# 2.把server程序放到tasks这个deque里
run()

# 终端1,运行aserver.py
# recv sock
$python3 -i aserver.py 
Connection ('127.0.0.1', 45601)
Connection ('127.0.0.1', 45602)
# 准备就绪,server接受client2 client3的连接
# recv client

# 终端2,运行client2
$nc localhost 25000
1
1
4
# client2 recv到4
# send client

# 终端3,运行client3
$python3 perf2.py
16493 reqs/sec
16197 reqs/sec
16377 reqs/sec
15675 reqs/sec
# 没有连接
0 reqs/sec
0 reqs/sec
0 reqs/sec
# 所以,可以发现,server被卡在recv client和send client之间

9.

socket.socketpair([family[, type[, proto]]])
# 创建一对连接的套接字对象
add_done_callback(fn)
# 在完成时，增加一个回调

10.待解决35

from socket import *
from fib import fib
from collections import deque
from select import select
from concurrent.futures import ThreadPoolExecutor as Pool
from concurrent.futures import ProcessPoolExecutor as Pool

pool = Pool(10)

tasks = deque()
recv_wait = { }         # Mapping sockets -> tasks (generators)
send_wait = { }
future_wait = { }

future_notify, future_event = socketpair()

def future_done(future):
    tasks.append(future_wait.pop(future))
    future_notify.send(b'x')

def future_monitor():
    while True:
        yield 'recv', future_event
        future_event.recv(100)

tasks.append(future_monitor())

def run():
    while any([tasks, recv_wait, send_wait]):
        while not tasks:
            # No active tasks to run
            # wait for I/O
            can_recv, can_send, _ = select(recv_wait, send_wait, [])
            for s in can_recv:
                tasks.append(recv_wait.pop(s))
            for s in can_send:
                tasks.append(send_wait.pop(s))
        task = tasks.popleft()
        try:
            why, what = next(task)      # Run to the yield
            if why == 'recv':
                # Must go wait somewhere
                recv_wait[what] = task
            elif why == 'send':
                send_wait[what] = task
            elif why == 'future':
                future_wait[what] = task
                what.add_done_callback(future_done)
            else:
                raise RuntimeError("ARG!")
        except StopIteration:
            print("task done")

def fib_server(address):
    sock = socket(AF_INET, SOCK_STREAM)
    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    sock.bind(address)
    sock.listen(5)
    while True:
        yield 'recv', sock
        client, addr = sock.accept() # blocking
        print("Connection", addr)
        tasks.append(fib_handler(client))

def fib_handler(client):
    while True:
        yield 'recv', client
        req = client.recv(100) # blocking
        if not req:
            break
        n = int(req)
        future = pool.submit(fib, n)  # 把要执行fib和参数n提交到线程池
        result = future.result()  # 取出线程池运行完的结果
        # result = fib(n)
        resp = str(result).encode('ascii') + b'\n'
        yield 'send', client
        client.send(resp)   # blocking
    print("Closed")

tasks.append(fib_server(('', 25000)))
run()

转载于:https://my.oschina.net/charlock/blog/861483