python3协程原理_python3协程学习

工作过程，参考python官方手册实现了一个socket长连接的客户端，使用了asyncio库中的streamreader，实现过程中着实费了一番功夫，虽然工作交差了，但感觉不好，还是想看看网上有没有学习视频，在B站找到这个，认为A. Jesse Jiryu Davis这位python大神的视频讲解，极为通俗易懂，

协程学习

copy视频例子学习阶段

服务端

也不知道视频中怎么实现，偷懒用flask实现一个超级简单的server端from flask import Flask

app = Flask(__name__)

@app.route("/")

def hello():

return "hello haha"

@app.route("/foo")

def foo():

return "=foo==" * 100

@app.route("/bar")

def bar():

return "=bar==" * 100

if __name__ == "__main__":

app.run()

执行方法：python3 flask_server.py

执行效果：ubuntu@VM-0-13-ubuntu:~$ python3 flask_server.py

* Serving Flask app "flask_server" (lazy loading)

* Environment: production

WARNING: This is a development server. Do not use it in a production deployment.

Use a production WSGI server instead.

* Debug mode: off

* Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)

127.0.0.1 - - [28/Nov/2020 20:08:25] "GET /foo HTTP/1.0" 200 -

socket客户端(v1.0)

知识点：selector，非阻塞socketfrom selectors import DefaultSelector,EVENT_WRITE,EVENT_READ

import socket

import time

selector = DefaultSelector()

def get(path):

s = socket.socket()

s.setblocking(False)

try:

s.connect(('127.0.0.1',5000))

except BlockingIOError:

pass

request = 'GET %s HTTP/1.0\r\n\r\n' % path

#可写

selector.register(s.fileno(), EVENT_WRITE)

selector.select()

selector.unregister(s.fileno())

s.send(request.encode())

chunks = []

while True:

#可读

selector.register(s.fileno(), EVENT_READ)

selector.select()

selector.unregister(s.fileno())

chunk = s.recv(10)

if chunk:

print("recive chunk:%s" % chunk)

chunks.append(chunk)

else:

body = (b''.join(chunks)).decode()

print(body.split('\n')[0])

return

start = time.time()

for i in range(10):

get('/foo')

get('/bar')

print('took %.2f sec' % (time.time()-start))

执行效果...

recive chunk:b'ar===bar=='

HTTP/1.0 200 OK

took 0.11 sec

socket客户端(v1.1)

知识点：回调函数 callbackfrom selectors import DefaultSelector,EVENT_WRITE,EVENT_READ

import socket

import time

selector = DefaultSelector()

n_tasks = 0

def get(path):

global n_tasks

s = socket.socket()

s.setblocking(False)

try:

s.connect(('127.0.0.1',5000))

except BlockingIOError:

pass

request = 'GET %s HTTP/1.0\r\n\r\n' % path

callback = lambda: connected(s,request)

selector.register(s.fileno(),EVENT_WRITE,data=callback)

n_tasks +=1

def connected(s,request):~~~~

selector.unregister(s.fileno())

# s可写

s.send(request.encode())

chunks = []

callback = lambda :readable(s,chunks)

selector.register(s.fileno(), EVENT_READ,data=callback)

def readable(s,chunks):

global n_tasks

selector.unregister(s.fileno())

chunk = s.recv(100)

if chunk:

print("recive chunk:%s" % chunk)

chunks.append(chunk)

callback = lambda: readable(s, chunks)

selector.register(s.fileno(), EVENT_READ,data=callback)

n_tasks -= 1

else:

body = (b''.join(chunks)).decode()

print(body.split('\n')[0])

return

start = time.time()

for i in range(10)

get('/foo')

get('/bar')

while n_tasks:

events = selector.select()

for event,mask in events:

cb = event.data

cb()

print('took %.2f sec' % (time.time()-start))

socket客户端(v1.2)

知识点：future对象from selectors import DefaultSelector,EVENT_WRITE,EVENT_READ

import socket

import time

selector = DefaultSelector()

n_tasks = 0

class Future:

def __init__(self):

self.callbacks = []

def resolve(self):

for c in self.callbacks:

c()

def get(path):

global n_tasks

s = socket.socket()

s.setblocking(False)

try:

s.connect(('127.0.0.1',5000))

except BlockingIOError:

pass

request = 'GET %s HTTP/1.0\r\n\r\n' % path

callback = lambda: connected(s,request)

fut = Future()

fut.callbacks.append(callback)

selector.register(s.fileno(),EVENT_WRITE,data=fut)

n_tasks +=1

def connected(s,request):

selector.unregister(s.fileno())

# s可写

s.send(request.encode())

chunks = []

callback = lambda :readable(s,chunks)

fut = Future()

fut.callbacks.append(callback)

selector.register(s.fileno(), EVENT_READ,data=fut)

def readable(s,chunks):

global n_tasks

selector.unregister(s.fileno())

chunk = s.recv(100)

if chunk:

print("recive chunk:%s" % chunk)

chunks.append(chunk)

callback = lambda: readable(s, chunks)

fut = Future()

fut.callbacks.append(callback)

selector.register(s.fileno(), EVENT_READ,data=fut)

n_tasks -= 1

else:

body = (b''.join(chunks)).decode()

print(body.split('\n')[0])

return

start = time.time()

for i in range(10)

get('/foo')

get('/bar')

while n_tasks:

events = selector.select()

for event,mask in events:

fut = event.data

fut.resolve()

print('took %.2f sec' % (time.time()-start))

socket客户端(v1.3)

知识点：Task，协程from selectors import DefaultSelector,EVENT_WRITE,EVENT_READ

import socket

import time

selector = DefaultSelector()

n_tasks = 0

class Future:

def __init__(self):

self.callbacks = []

def resolve(self):

for c in self.callbacks:

c()

class Task:

def __init__(self,gen):

self.gen=gen

self.step()

def step(self):

try:

f = next(self.gen)

except StopIteration:

return

f.callbacks.append(self.step)

def get(path):

global n_tasks

n_tasks +=1

s = socket.socket()

s.setblocking(False)

try:

s.connect(('127.0.0.1',5000))

except BlockingIOError:

pass

request = 'GET %s HTTP/1.0\r\n\r\n' % path

fut = Future()

selector.register(s.fileno(), EVENT_WRITE,data=fut)

yield fut

selector.unregister(s.fileno())

s.send(request.encode())

chunks = []

while True:

fut = Future()

selector.register(s.fileno(), EVENT_READ,data=fut)

yield fut

selector.unregister(s.fileno())

chunk = s.recv(100)

if chunk:

print("recive chunk:%s" % chunk)

chunks.append(chunk)

else:

body = (b''.join(chunks)).decode()

print(body.split('\n')[0])

n_tasks-=1

start = time.time()

for i in range(10)

Task(get('/foo'))

Task(get('/bar'))

while n_tasks:

events = selector.select()

for event,mask in events:

fut = event.data

fut.resolve()

print('took %.2f sec' % (time.time()-start))

ECHO小例子，server，client

server端：import socket

import asyncio

from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ

class Server(object):

MAX_BUF = 1024

def __init__(self, port):

self._client = []

self._loop = asyncio.new_event_loop()

self._selector = DefaultSelector()

self._sock = self.socket_init()

def socket_init(self):

sock = socket.socket()

sock.bind(('127.0.0.1',5000))

sock.listen(Server.MAX_BUF)

self._selector.register(sock,EVENT_READ,self.accept_client)

return sock

def accept_client(self,sock,mask):

conn, addr = sock.accept()

print(f"recv a conn:{conn} from {addr}")

self._client.append(conn)

print(f"now has client:{self._client}")

conn.setblocking(False)

self._selector.register(conn, EVENT_READ, self.read_from_client)

def read_from_client(self, conn, mask):

data = conn.recv(Server.MAX_BUF)

if data:

print(f"recv data from client:{data}")

conn.send(data) # echo first todo

import time

time.sleep(10)

else:

self._selector.unregister(conn)

conn.close()

def run_forever(self):

while True:

events = self._selector.select()

for event,mask in events:

cb = event.data

cb(event.fileobj,mask)

if __name__ == '__main__':

ser = Server(5000)

ser.run_forever()

client：import asyncio

from concurrent.futures import TimeoutError

async def tcp_echo_client(message):

reader, writer = await asyncio.open_connection(

'127.0.0.1', 5000)

print(f'Send: {message!r}')

writer.write(message.encode())

while not reader.at_eof():

print(type(reader))

print(reader)

print(type(writer))

print(writer)

try:

writer.write(message.encode())

data=await asyncio.wait_for(reader.read(1), 2)

print(f'Received: {data.decode()!r}')

except TimeoutError:

import traceback

print("%s"% traceback.format_exc())

asyncio.get_event_loop().run_until_complete(tcp_echo_client('Hello World!'))

参考- [1] [一个使用 asyncio 协程的网络爬虫](https://linux.cn/article-8265-1.html)

- [2] [Python协程工作原理](https://www.bilibili.com/video/BV1N4411S7BP?t=32&p=2)