python多线程下载图片

---

一、pandas是什么？

pandas 是基于NumPy 的一种工具，该工具是为了解决数据分析任务而创建的。

二、使用步骤

1.引入库

代码如下（示例）：

import pandas as pd
import requests
import os
import concurrent.futures
from loguru import logger

2.读入数据,生成jpg文件

代码如下：

def get_list(dir):
    """
    获取imglist和namelist 用于下载图片和生成图片名称
    :param dir:
    :return:
    """
    #读取excel文件
    df = pd.read_excel(dir)
    #根据某一列去去重
    df = df.drop_duplicates(subset=['xxx'],keep='first')
    img_list = df['xxx'].tolist()
    name_list = df['xxx'].tolist()
    return img_list,name_list 
    
def download_img(i,down_dir,img_list,UPC_list):
    """
    下载图片到本地目录
    :param i:
    :param down_dir:
    :param img_list:
    :param UPC_list:
    :return:
    """
    resp = requests.get(img_list[i])
    # 获取的文本实际上是图片的二进制文本
    img = resp.content
    img_name = str(name_list[i])
    img_path = os.path.join(down_dir, img_name)
    # 将他拷贝到本地文件 w 写  b 二进制  wb代表写入二进制文本
    logger.info(img_name,'正在生成。。。')
    with open(img_path + '.jpg', 'wb') as f:
        f.write(img)







def main():
    # https://xxx
    # img_url_list=[xxx,xxx,xxx] 
    # name_list=[xxx,xxx,xxx]
    img_list,name_list = get_list(dir)
    # We can use a with statement to ensure threads are cleaned up promptly,0 max_workers is ThreadNum
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        # Start the load operations and mark each future with its URL
        future_to_url = {executor.submit(download_img, i,down_dir,img_list,name_list): i for i in range(len(img_list))}
        for future in concurrent.futures.as_completed(future_to_url):
            number = future_to_url[future]
            url = img_list[number]
            name= name_list[number]
            try:
                data = future.result()
            except Exception as exc:
                logger.info(f'{name} {url} generated an exception: {exc}')
            else:
                logger.info(f'{name} {url} success download')



if __name__ == '__main__':
    dir = r'xxx'
    down_dir = r'xxx'
    main()

该处使用的url网络请求的数据。

---

总结

可以通过修改线程数来控制所有图片下载的速度