python 异步协程下载图片

Awesome Wallpapers - wallhaven.cc

今天用到的网站是一个壁纸网站，图片精美。

import requests
import re
import os
import asyncio
import aiohttp
import aiofiles


def gethtml(url, encode):  # 获取网页源码
    r = requests.get(url)
    r.encoding = encode
    return r.text



def mkdir(path):  # 创建文件夹
    path = path.strip()
    isExists = os.path.exists(path)
    if not isExists:
        os.makedirs(path)


def geturl(url):  # 获取网站各个分类链接、分类名称
    html = gethtml(url, 'utf-8')
    obj = re.compile(r'<a class="preview" href="(?P<url>.*?)"  target="_blank"  ></a>', re.S)
    result = obj.finditer(html)
    with open('image_path.txt', 'w') as f:
        for item in result:
            f.write(item.group('url') + '\n')


async def aio_download_pic(session, img_url):
    pic_name = img_url.split('/')[-1]
    img_url = get_data(img_url)
    print(img_url)
    async with session.get(img_url) as resp:
        async with aiofiles.open(save_path + pic_name+'.jpg', mode='wb') as f:
            await f.write(await resp.content.read())
    print(pic_name + "下载完毕")


def get_data(page_url):
    page_content = gethtml(page_url, 'utf-8')
    obj = re.compile(r'<img id="wallpaper" src="(?P<img_true_url>.*?)"', re.S)
    img_true_url = obj.search(page_content, re.S).group('img_true_url')
    return img_true_url


async def aio_download():
    tasks = []
    timeout = aiohttp.ClientTimeout(total=600)
    async with aiohttp.ClientSession(timeout=timeout) as session:
        async with aiofiles.open('image_path.txt', mode='r', encoding='utf-8') as f:
            async for line in f:
                print(line)
                url = line.strip()
                tasks.append(asyncio.create_task(aio_download_pic(session, url)))
            await asyncio.wait(tasks)


if __name__ == '__main__':
    base_url = 'https://wallhaven.cc/toplist?page=2'
    save_path = 'E:\\wall image\\'  # 保存位置
    mkdir(save_path)
    geturl(base_url)
    """
        多线程下载图片
    """
    # try:
    #     asyncio.run(aio_download())
    # except:
    #     pass

    loop = asyncio.get_event_loop()
    loop.run_until_complete(aio_download())