python3.9异步同时下载多个文件（带下载进度条）

需求场景

使用python同时下载多个文件（带下载进度条）

代码实现

使用异步任务，完成该需求，python版本：3.9

import aiohttp
import asyncio
import traceback
from tqdm.asyncio import tqdm
from loguru import logger

# 设置同时下载的最大文件数量
concurrent_downloads = 5
semaphore = asyncio.Semaphore(concurrent_downloads)

# 核心下载文件方法
async def download_file(url, save_path):
    try:
        async with semaphore:
            timeout = aiohttp.ClientTimeout(10 * 60)
            async with aiohttp.ClientSession(timeout=timeout) as session:
                async with session.get(url) as response:
                    if response.status != 200:
                        raise Exception(f"Failed to download file: {url}, status: {response.status}")
                    total_size = int(response.headers.get('Content-Length', 0))
                    with open(save_path, 'wb') as f:
                        with tqdm(total=total_size, desc=save_path, unit='B', unit_scale=True) as pbar:
                            while True:
                                chunk = await response.content.read(1024)
                                if not chunk:
                                    break
                                f.write(chunk)
                                pbar.update(len(chunk))
            return save_path
    except asyncio.TimeoutError as e:
        logger.debug(f"下载请求超时 {url}: {e}")
        return None
    except Exception as e:
        tb = e.__traceback__
        last_line = traceback.format_tb(tb)[-1]
        logger.debug(f"Error downloading {url}: {e}-{last_line}")
        return None

# 异步下载,可并行下载多个（对外调用）
async def download(files_to_download: list):
    """
    接收参数
     [
        {
            'url': 'https://xxxx.zip',
            'save_path': xxx
        },
    ]
    """
    tasks = [
        download_file(file['url'], file['save_path']) for file in files_to_download
    ]

    downloaded_files = await asyncio.gather(*tasks)
    for downloaded_file in downloaded_files:
        if downloaded_file is not None:
            logger.debug('文件下载成功 {}'.format(downloaded_file))
        else:
            logger.error("文件下载失败")
            raise Exception("文件下载失败，请检查链接地址是否正确！")

外部调用

from xxx import download

# 第一种调用方式
asyncio.run(download(files_to_download))
# 第二种调用方式
loop = asyncio.get_event_loop()
loop.run_until_complete(download(files_to_download))