Python 协程 全面剖析

引入

单纯地切换反而会降低运行效率

# 串行执行
import time


def consumer(res):
    '''任务1:接收数据,处理数据'''
    pass


def producer():
    '''任务2:生产数据'''
    res = []
    for i in range(10000000):
        res.append(i)
    return res


start = time.time()
# 串行执行
res = producer()
consumer(res)  # 写成consumer(producer())会降低执行效率
stop = time.time()
print(stop - start)  # 1.5536692142486572

# 基于yield并发执行
import time


def consumer():
    '''任务1:接收数据,处理数据'''
    while True:
        x = yield


def producer():
    '''任务2:生产数据'''
    g = consumer()
    next(g)
    for i in range(10000000):
        g.send(i)


start = time.time()
# 基于yield保存状态,实现两个任务直接来回切换,即并发的效果
# PS:如果每个任务中都加上打印,那么明显地看到两个任务的打印是你一次我一次,即并发执行的.
producer()

stop = time.time()
print(stop - start)  # 2.0272178649902344

第一种情况的切换。在任务一遇到io情况下，切到任务二去执行，这样就可以利用任务一阻塞的时间完成任务二的计算，效率的提升就在于此。

# yield无法做到遇到io阻塞

import time


def consumer():
    '''任务1:接收数据,处理数据'''
    while True:
        x = yield


def producer():
    '''任务2:生产数据'''
    g = consumer()
    next(g)
    for i in range(10000000):
        g.send(i)
        time.sleep(2)


start = time.time()
producer()  
# 并发执行,但是任务producer遇到io就会阻塞住,并不会切到该线程内的其他任务去执行

stop = time.time()
print(stop - start)

协程的介绍

进程，操作系统中存在；
线程，操作系统中存在；
协程，是由程序员创造出来的一个不是真实存在的东西；
协程：是微线程，对一个线程进程分片，使得线程在代码块之间进行来回切换执行，
而不是在原来逐行执行。

单纯的线程毫无意义，甚至还会损害性能↓↓↓

import greenlet

def f1():
     print(11)
     gr2.switch()
     print(22)
     gr2.switch()


def f2():
     print(33)
     gr1.switch()
     print(44)

gr1=greenlet.greenlet(f1)
gr2=greenlet.greenlet(f2)

gr1.switch()

遇到IO操作就切换>>>就会变得有意义

Greenlet模块

安装 ：pip3 install greenlet

# greenlet实现状态切换

from greenlet import greenlet


def eat(name):
    print('%s eat 1' % name)
    g2.switch('egon')
    print('%s eat 2' % name)
    g2.switch()


def play(name):
    print('%s play 1' % name)
    g1.switch()
    print('%s play 2' % name)


g1 = greenlet(eat)
g2 = greenlet(play)

g1.switch('egon')  # 可以在第一次switch时传入参数，以后都不需要

'''
单纯的切换（在没有io的情况下或者没有重复开辟内存空间的操作），
--反而会降低程序的执行速度
'''

效率对比

# 顺序执行
import time


def f1():
    res = 1
    for i in range(100000000):
        res += i


def f2():
    res = 1
    for i in range(100000000):
        res *= i


start = time.time()
f1()
f2()
stop = time.time()
print('run time is %s' % (stop - start))  # 10.985628366470337

# 切换
from greenlet import greenlet
import time


def f1():
    res = 1
    for i in range(100000000):
        res += i
        g2.switch()


def f2():
    res = 1
    for i in range(100000000):
        res *= i
        g1.switch()


start = time.time()
g1 = greenlet(f1)
g2 = greenlet(f2)
g1.switch()
stop = time.time()
print('run time is %s' % (stop - start))  # 52.763017892837524

Gevent模块

Gevent的使用

用法介绍

'''
g1=gevent.spawn(func,1,,2,3,x=4,y=5)创建一个协程对象g1，
	spawn括号内第一个参数是函数名，如eat，后面可以有多个参数，
	可以是位置实参或关键字实参，都是传给函数eat的

g2=gevent.spawn(func2)

g1.join() #等待g1结束

g2.join() #等待g2结束

#或者上述两步合作一步：gevent.joinall([g1,g2])

g1.value#拿到func1的返回值
'''

实例：遇到io主动切换

import gevent


def eat(name):
    print('%s eat 1' % name)
    gevent.sleep(2)
    print('%s eat 2' % name)


def play(name):
    print('%s play 1' % name)
    gevent.sleep(1)
    print('%s play 2' % name)


g1 = gevent.spawn(eat, 'jay')
g2 = gevent.spawn(play, name='jay')
g1.join()
g2.join()
# 或者gevent.joinall([g1,g2])
print('主')

'''
jay eat 1
jay play 1
jay play 2
jay eat 2
主
'''

from gevent import monkey;

monkey.patch_all()

import gevent
import time


def eat():
    print('eat food 1')
    time.sleep(2)
    print('eat food 2')


def play():
    print('play 1')
    time.sleep(1)
    print('play 2')


g1 = gevent.spawn(eat)
g2 = gevent.spawn(play)
gevent.joinall([g1, g2])
print('主')
'''
eat food 1
play 1
play 2
eat food 2
主
'''

查看threading.current_thread().getName()

from gevent import monkey;

monkey.patch_all()
import threading
import gevent
import time


def eat():
    print(threading.current_thread().getName())
    print('eat food 1')
    time.sleep(2)
    print('eat food 2')


def play():
    print(threading.current_thread().getName())
    print('play 1')
    time.sleep(1)
    print('play 2')


g1 = gevent.spawn(eat)
g2 = gevent.spawn(play)
gevent.joinall([g1, g2])
print('主')
'''
DummyThread-1
eat food 1
DummyThread-2
play 1
play 2
eat food 2
主
'''

实例

from gevent import monkey;monkey.patch_all()  
# 以后代码中遇到IO都会自动执行greenlet的switch进行切换
import requests
import gevent


def get_page1(url):
    ret = requests.get(url)
    print(url, ret.content)


def get_page2(url):
    ret = requests.get(url)
    print(url, ret.content)


def get_page3(url):
    ret = requests.get(url)
    print(url, ret.content)


gevent.joinall([
    gevent.spawn(get_page1, 'https://www.python.org/'),  # 协程1
    gevent.spawn(get_page2, 'https://www.yahoo.com/'),  # 协程2
    gevent.spawn(get_page3, 'https://github.com/'),  # 协程3
])

from gevent import monkey;monkey.patch_all()
import gevent
import urllib2


def f(url):
    print('GET: %s' % url)
    resp = urllib2.urlopen(url)
    data = resp.read()
    print('%d bytes received from %s.' % (len(data), url))


gevent.joinall([
    gevent.spawn(f, 'https://www.python.org/'),
    gevent.spawn(f, 'https://www.yahoo.com/'),
    gevent.spawn(f, 'https://github.com/'),
])

总结

'''
总结：
1. 什么是协程？
协程也可以称为“微线程”，就是开发者控制线程执行流程，
控制先执行某段代码然后再切换到另外函执行代码...来回切换。

2. 协程可以提高并发吗？
协程自己本身无法实现并发（甚至性能会降低）。
协程+IO切换性能提高。

3. 进程、线程、协程的区别？

4. 单线程提供并发：
- 协程+IO切换：gevent
- 基于事件循环的异步非阻塞框架：Twisted
'''

Gevent之同步与异步

from gevent import spawn, joinall, monkey;monkey.patch_all()

import time


def task(pid):
    """
    Some non-deterministic task
    """
    time.sleep(0.5)
    print('Task %s done' % pid)


def synchronous():  # 同步
    for i in range(10):
        task(i)


def asynchronous():  # 异步
    g_l = [spawn(task, i) for i in range(10)]
    joinall(g_l)
    print('DONE')


if __name__ == '__main__':
    print('Synchronous:')
    synchronous()
    print('Asynchronous:')
    asynchronous()
#  上面程序的重要部分是将task函数封装到Greenlet内部线程的gevent.spawn。
#  初始化的greenlet列表存放在数组threads中，此数组被传给gevent.joinall 函数，
#  后者阻塞当前流程，并执行所有给定的greenlet任务。执行流程只会在
#  --所有greenlet执行完后才会继续向下走。
'''
Synchronous:
Task 0 done
Task 1 done
Task 2 done
Task 3 done
Task 4 done
Task 5 done
Task 6 done
Task 7 done
Task 8 done
Task 9 done
Asynchronous:
Task 0 done
Task 1 done
Task 2 done
Task 3 done
Task 4 done
Task 5 done
Task 6 done
Task 7 done
Task 8 done
Task 9 done
DONE
'''

Gevent的应用

# server端

from gevent import monkey;monkey.patch_all()
from socket import *
import gevent


# 如果不想用money.patch_all()打补丁,可以用gevent自带的socket
# from gevent import socket
# s=socket.socket()

def server(server_ip, port):
    s = socket(AF_INET, SOCK_STREAM)
    s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
    s.bind((server_ip, port))
    s.listen(5)
    while True:
        conn, addr = s.accept()
        gevent.spawn(talk, conn, addr)


def talk(conn, addr):
    try:
        while True:
            res = conn.recv(1024)
            print('client %s:%s msg: %s' % (addr[0], addr[1], res))
            conn.send(res.upper())
    except Exception as e:
        print(e)
    finally:
        conn.close()


if __name__ == '__main__':
    server('127.0.0.1', 8080)

# client端

from socket import *

client = socket(AF_INET, SOCK_STREAM)
client.connect(('127.0.0.1', 8080))

while True:
    msg = input('>>: ').strip()
    if not msg: continue

    client.send(msg.encode('utf-8'))
    msg = client.recv(1024)
    print(msg.decode('utf-8'))

# 多线程并发多个客户端

from threading import Thread
from socket import *
import threading


def client(server_ip, port):
    c = socket(AF_INET, SOCK_STREAM)  
    # 套接字对象一定要加到函数内，即局部名称空间内，放在函数外则被所有线程共享，
    # --则大家公用一个套接字对象，那么客户端端口永远一样了
    c.connect((server_ip, port))

    count = 0
    while True:
        c.send(('%s say hello %s' % (threading.current_thread().getName(), count)).encode('utf-8'))
        msg = c.recv(1024)
        print(msg.decode('utf-8'))
        count += 1


if __name__ == '__main__':
    for i in range(500):
        t = Thread(target=client, args=('127.0.0.1', 8080))
        t.start()