python 异步协程的使用

目标

消费流(kafka或者queue)的数据，使用python协程异步处理这些数据，提高python的并发性

理解协程异步

协程理解起来很简单，但是用起来费劲，结合实际的项目，下面给出一个简单的协程实力

这里使用yield生成协程的生成器

import datetime
import random
import time
from multiprocessing import Queue

import asyncio


class worker(object):
    def __init__(self, stream=None):
        self.coro_work = None
        self.stream = stream

    def run(self):
        self.coro_work = self.worker()
        next(self.coro_work)
        self.feeder()

    def worker(self):
        while True:
            data = yield
            if data:
                sleep_time = random.random() / 100
                self.do_worker(data, sleep_time)
            else:
                print(datetime.datetime.now(), 'worker get None then stop.')

    def feeder(self):
        while True:
            if self.stream.qsize() == 0:
                self.close()
                break
            else:
                data = self.stream.get(block=True, timeout=1)
            print(datetime.datetime.now(), 'do worker on {:5d}'.format(data))
            self.coro_work.send(data)

    def do_worker(self, data, sleep_time):
        # 这里asyncio.sleep仅仅为了模拟需要协程处理的场景，用sleep没法触发协程处理
        asyncio.sleep(sleep_time)
        print(datetime.datetime.now(), 'do worker on {:5d} sleep {:.6f}'.format(data, sleep_time))

    def close(self):
        self.stream.close()
        self.coro_work.close()


if __name__ == '__main__':
    # sleep_time = random.random()
    # print(sleep_time)

    stream = Queue(maxsize=1000)
    for i in range(0, 1000):
        stream.put(i)
    kk = stream.get(block=True, timeout=1)
    print(kk)
    #
    # time.sleep(1)
    work_begin = time.time()
    worker(stream).run()
    work_finish = time.time()
    print('time cost:', work_finish - work_begin)

对比 yield from和asyncio

yield from 的场景

asyncio 的场景

参考链接

1. 刘江的博客教程-协程与异步IO
2. Go 协程实现原理和使用示例