解决新浪微博图床403批量下载图片等资源（以MMChat数据集为例）

---

1. 代码

该 Python 脚本可多线程地批量下载新浪图床图片，每次下载会检查哪些图片已下载并过滤已下载的图片。

import os
import requests
from concurrent.futures import ThreadPoolExecutor, as_completed
import logging
import time

from tqdm import tqdm


def setup_logger():
    # 设置日志记录：同时输出到文件和控制台
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)

    # 文件处理器
    file_handler = logging.FileHandler('download.log')
    file_handler.setLevel(logging.INFO)

    # 控制台处理器
    console_handler = logging.StreamHandler()
    console_handler.setLevel(logging.ERROR)

    # 日志格式
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    file_handler.setFormatter(formatter)
    console_handler.setFormatter(formatter)

    # 添加处理器到logger
    logger.addHandler(file_handler)
    logger.addHandler(console_handler)


def download_image_by_change_referer(output_dir, url):
    """
    下载图片
    Args:
        output_dir:     保存图片的文件夹
        url:            图片链接

    Returns:    下载成功返回True，否则返回False

    """
    headers = {
        'Referer': 'https://weibo.com/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
    }

    try:
        response = requests.get(url, headers=headers, timeout=10)
        if response.status_code == 200:
            file_name = os.path.join(output_dir, url.split('/')[-1])
            with open(file_name, 'wb') as f:
                f.write(response.content)
            logging.info(f'Successfully downloaded {file_name}')
            return True
        else:
            logging.error(f'Failed to download {url}, status code: {response.status_code}')
            return False
    except requests.RequestException as e:
        logging.error(f'Error downloading {url}: {e}')
        return False
    # time.sleep(0.1)  # 限制请求频率，防止过快请求


def download_image_by_baidu_cache(output_dir, url):
    """
    使用第三方缓存服务来解决防盗链问题
    参考 https://code.newban.cn/466.html
    Args:
        output_dir:     保存图片的文件夹
        url:            图片链接

    Returns:            下载成功返回True，否则返回False

    """
    headers = {
        'Referer': 'https://image.baidu.com/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
    }
    try:
        response = requests.get(f'https://image.baidu.com/search/down?url={url}', headers=headers, timeout=10)
        if response.status_code == 200:
            file_name = os.path.join(output_dir, url.split('/')[-1])
            with open(file_name, 'wb') as f:
                f.write(response.content)
            logging.info(f'Successfully downloaded {file_name}')
            return True
        else:
            logging.error(f'Failed to download {url}, status code: {response.status_code}')
            return False
    except requests.RequestException as e:
        logging.error(f'Error downloading {url}: {e}')
        return False


def batch_download(output_dir, urls, method='baidu_cache', max_workers=10):
    """
    批量下载图片
    Args:
        output_dir:     保存图片的文件夹
        urls:           图片链接列表
        max_workers:    最大线程数
        method:         下载方法，可选值为'change_referer'或'baidu_cache'

    Returns:            成功下载的图片数量和失败的图片数量

    """
    if method == 'change_referer':
        download_image_function = download_image_by_change_referer
    elif method == 'baidu_cache':
        download_image_function = download_image_by_baidu_cache
    else:
        raise ValueError(f'Invalid method: {method}')
    success_count, failed_count = 0, 0
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        future_to_url = {executor.submit(download_image_function, output_dir, url): url for url in urls}
        for future in tqdm(as_completed(future_to_url), total=len(urls)):
            url = future_to_url[future]
            try:
                result = future.result()
                if result:
                    success_count += 1
                else:
                    failed_count += 1
            except Exception as e:
                logging.error(f'Error processing {url}: {e}')
                failed_count += 1
    return success_count, failed_count


def get_all_image_urls(dataset_root_dir):
    """
    读取所有图片链接
    Args:
        dataset_root_dir:   数据集根目录，包含img_url_dev.json, img_url_test.json, img_url_train.json
    Returns:    图片链接列表

    """
    files = [r'img_url_dev.json', r'img_url_test.json', r'img_url_train.json']
    image_urls = set()
    for idx, file in enumerate(files):
        path = os.path.join(dataset_root_dir, file)
        with open(path, 'r', encoding='utf-8') as f:
            lines = f.readlines()
        for line in lines:
            image_urls.update(line.strip().split(';'))
    return image_urls


def get_downloaded_images(output_dir):
    """
    读取已下载的图片（避免重复下载）
    Returns:    已下载的图片集合

    """
    downloaded_images = os.listdir(output_dir)
    return downloaded_images


def main():
    setup_logger()
    config = dict(
        dataset_root_dir=r'D:\Library\Datasets\20_MMChat',
        output_dir='downloaded_images',
        download_image_method='baidu_cache',
    )

    # 图片链接列表
    image_urls = get_all_image_urls(config['dataset_root_dir'])
    # 目标文件夹
    output_dir = config['output_dir']
    os.makedirs(output_dir, exist_ok=True)
    # 过滤已下载的图片
    downloaded_images = set(get_downloaded_images(output_dir))
    image_urls_to_download = [url for url in image_urls if url.split('/')[-1] not in downloaded_images]
    logging.info(f'Total images: {len(image_urls)}, images to download: {len(image_urls_to_download)}')
    confirm = input(
        f'共有 {len(image_urls)} 张图片，已过滤 {len(image_urls) - len(image_urls_to_download)} 张已下载的图片，待下载 {len(image_urls_to_download)} 张图片，确认下载？(y/n): ')
    if confirm.lower() != 'y':
        return

    # 开始批量下载
    success_count, failed_count = batch_download(output_dir, image_urls_to_download,
                                                 method=config['download_image_method'], max_workers=40)

    logging.info(f'下载完成，Success: {success_count}, Failed: {failed_count}')
    print(f'下载完成，Success: {success_count}, Failed: {failed_count}')


if __name__ == '__main__':
    main()

2. 举一反三

该代码适用 MMChat 数据集，原理上支持所有新浪图床批量下载已失效的图片。

你需要修改 get_all_image_urls 方法来获取你想下载的所有图片的 URL 列表，该方法返回值：

[
    "https://wx2.sinaimg.cn/mw2048/bc5ca296ly1fpt9oq74vsj20hs0npdqp.jpg",
    "https://wx3.sinaimg.cn/mw2048/8bec28c2ly1fg1i9o0liqj20zk0qo7bj.jpg",
    "https://wx4.sinaimg.cn/mw2048/954d55d0ly1fmwvia87e1j20ku11210q.jpg",
    ...
]

get_downloaded_images 方法返回已下载的图片，如果上面的三张图片已下载，那么该方法会返回

[
    "bc5ca296ly1fpt9oq74vsj20hs0npdqp.jpg",
    "8bec28c2ly1fg1i9o0liqj20zk0qo7bj.jpg",
    "954d55d0ly1fmwvia87e1j20ku11210q.jpg",
]

最后，你可以修改图片的下载方式，目前支持 download_image_by_change_referer、download_image_by_baidu_cache。