asyncio 爬虫实例，只返回源码

最新推荐文章于 2022-01-04 18:22:20 发布

迷心兔

最新推荐文章于 2022-01-04 18:22:20 发布

阅读量175

点赞数

迷心兔

本文链接：https://blog.csdn.net/mixintu/article/details/102510895

版权

Python asyncio 专栏收录该内容

29 篇文章 2 订阅

订阅专栏

# coding=utf-8
import asyncio
import time
from aiohttp import ClientSession
from contextlib import closing


async def fetch(url):
    try:
        async with ClientSession() as session:
            async with session.get(url) as response:
                response = await response.read()
                # logging.info(response[-20:])
                return response[-20:]
    except asyncio.CancelledError:
        raise


async def main(event_loop_):
    now = time.time()
    # -------------------------------------------------------------------------
    tasks = []
    urls = [
        "https://docs.python.org/3/library/asyncio-task.html",
        "https://www.cnblogs.com/yzh2857/p/10376598.html",
        "https://www.cnblogs.com/yzh2857/p/10390808.html",
        "https://matplotlib.org/api/pyplot_api.html",
        "https://www.programcreek.com/python/index/module/list",
        "https://selenium-python.readthedocs.io/api.html",
        "https://www.w3resource.com/python/python-tutorial.php",
        "https://docs.scipy.org/doc/numpy/reference/generated/numpy.ones.html",
        "https://www.w3resource.com/python/module/calendar/",
    ]
    for url in urls:
        tasks.append(asyncio.ensure_future(fetch(url)))
    # 第一种运行方式-------------------------------------------------------------
    dones, pendings = await asyncio.wait(tasks)
    # 已完成的协程
    for done in dones:
        print("已完成的协程：", done)
    # 超时未完成的协程
    for pending in pendings:
        print("超时未完成的协程  ：", pending)
        pending.cancel()
    # 第二种运行方式-------------------------------------------------------------
    # result = await asyncio.gather(*tasks)
    # print("结果：", result)
    # 第三种运行方式---------------------------
    # for task in asyncio.as_completed(tasks):
    #     result = await task
    #     print("结果：", result)
    # ---------------------------------------------------------------------------
    print("总用时", time.time() - now)


if __name__ == "__main__":
    with closing(asyncio.get_event_loop()) as loop:
        loop.run_until_complete(main(loop))

输出：

已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'pt>\n</body>\n</html>\n'>
已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'iv>\n</body>\n</html>\n'>
已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'>\n\n  </body>\n</html>'>
已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'ript></body></html>\n'>
已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'  \n  </body>\n</html>'>
已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'iv>\n</body>\n</html>\n'>
已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'>\n\n</footer>\n</html>'>
已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'v>\n  </body>\n</html>'>
已完成的协程： <Task finished coro=<fetch() done, defined at C:\Users\Administrator\Desktop\tmp.py:8> result=b'pt>\n</body>\n</html>\n'>
总用时 3.107325315475464