背景
这两天需要用到python下载数据,用pandas读了之后发现单线程执行有点慢,所以改成多线程代码如下:
data = pd.read_sql(query, engine)
data.columns = ['BARCODE','pictrue_name','path','V_MONTH','V_DAY','V_HOUR']
i=1
def copy_file(row):
value1 = row[1]
source2 = str(row[2])
destination_folder = os.path.join("D:\\", row[3], row[4], row[0])
if not os.path.exists(destination_folder):
os.makedirs(destination_folder)
save_path = os.path.join(destination_folder, value1)
shutil.copy(source2, save_path)
with ThreadPoolExecutor(max_workers=8) as executor:
# 提交任务到线程池
futures = [executor.submit(copy_file, row) for row in data.itertuples(index=False)]
# 等待所有任务完成
for future in futures:
future.result()
多线程这一步比较简单,主要不和数据库有交互,python封装得挺好的,比最开始快了七八倍