python多线程下载图片
一、pandas是什么?
pandas 是基于NumPy 的一种工具,该工具是为了解决数据分析任务而创建的。
二、使用步骤
1.引入库
代码如下(示例):
import pandas as pd
import requests
import os
import concurrent.futures
from loguru import logger
2.读入数据,生成jpg文件
代码如下:
def get_list(dir):
"""
获取imglist和namelist 用于下载图片和生成图片名称
:param dir:
:return:
"""
#读取excel文件
df = pd.read_excel(dir)
#根据某一列去去重
df = df.drop_duplicates(subset=['xxx'],keep='first')
img_list = df['xxx'].tolist()
name_list = df['xxx'].tolist()
return img_list,name_list
def download_img(i,down_dir,img_list,UPC_list):
"""
下载图片到本地目录
:param i:
:param down_dir:
:param img_list:
:param UPC_list:
:return:
"""
resp = requests.get(img_list[i])
# 获取的文本实际上是图片的二进制文本
img = resp.content
img_name = str(name_list[i])
img_path = os.path.join(down_dir, img_name)
# 将他拷贝到本地文件 w 写 b 二进制 wb代表写入二进制文本
logger.info(img_name,'正在生成。。。')
with open(img_path + '.jpg', 'wb') as f:
f.write(img)
def main():
# https://xxx
# img_url_list=[xxx,xxx,xxx]
# name_list=[xxx,xxx,xxx]
img_list,name_list = get_list(dir)
# We can use a with statement to ensure threads are cleaned up promptly,0 max_workers is ThreadNum
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
# Start the load operations and mark each future with its URL
future_to_url = {executor.submit(download_img, i,down_dir,img_list,name_list): i for i in range(len(img_list))}
for future in concurrent.futures.as_completed(future_to_url):
number = future_to_url[future]
url = img_list[number]
name= name_list[number]
try:
data = future.result()
except Exception as exc:
logger.info(f'{name} {url} generated an exception: {exc}')
else:
logger.info(f'{name} {url} success download')
if __name__ == '__main__':
dir = r'xxx'
down_dir = r'xxx'
main()
该处使用的url网络请求的数据。
总结
可以通过修改线程数来控制所有图片下载的速度