python:多协程编程

协程的两个案例

多协程可以实现异步操作，将IO异步，这样大大节省了IO的开销，我们这里还是举两个例子

这里用到了一个新的库 aiohttp

import asyncio
import aiohttp
from spider import urls

# 并不是什么库都支持协程，requests就不支持
async def async_craw(url):
    print('craw url:',url)
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as resp:
            result = await resp.text()
            print(f'craw url:{url},{len(result)}')

loop = asyncio.get_event_loop()

tasks = [loop.create_task(async_craw(url)) for url in urls]

loop.run_until_complete(asyncio.wait(tasks))
print('end')

import asyncio
import aiohttp
from spider import urls

semaphore = asyncio.Semaphore(10)

# 并不是什么库都支持协程，requests就不支持
async def async_craw(url):
    async with semaphore:
        print('craw url:',url)
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as resp:
                result = await resp.text()
                print(f'craw url:{url},{len(result)}')

loop = asyncio.get_event_loop()

tasks = [loop.create_task(async_craw(url)) for url in urls]

loop.run_until_complete(asyncio.wait(tasks))
print('end')