requests + asyncio 异步爬虫

import requests
import os
import asyncio

async def asyncget(url,headers):
    return requests.get(url,headers=headers,verify=False)

async def download(url,headers,fileDir,filename=None):

    if filename == None:
        filename = url[url.rfind("/")+1:]
    if fileDir == None:
        fileDir = "./files"

    if not os.path.exists(fileDir):
        print("亲，输入的目录不存在哦，下面将自动创建 "+ fileDir)
        os.makedirs(fileDir)
    
    filepath = os.path.join(fileDir,filename)

    if os.path.exists(filepath):
        print("亲，文件",filepath,"已经存在哦")
        return
    
    response =  await asyncget(url,headers=headers)

    with open(filepath,"wb") as fp:
        fp.write(response.content)
    print("亲，文件",filepath,"写入完成哦")


if __name__ == "__main__":
    headers ={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
    }
    urls = [
        "http://wx1.sinaimg.cn/mw600/00893JKXly1gikm1th96cj30u015b1a9.jpg",
        "http://wx1.sinaimg.cn/mw600/00893JKXly1gikm1lkcxdj30sg18g0zh.jpg",
        "http://wx2.sinaimg.cn/mw600/005QrDFkly1gfl4phpqvaj31w91w9x20.jpg"
    ]

    asyncio.run(
        asyncio.wait(
            [download(url=url,headers=headers,fileDir="imgs") for url in urls]
        )
    )