使用多线程读取pandas

背景

 这两天需要用到python下载数据，用pandas读了之后发现单线程执行有点慢，所以改成多线程代码如下：

data = pd.read_sql(query, engine)
data.columns = ['BARCODE','pictrue_name','path','V_MONTH','V_DAY','V_HOUR']
i=1


def copy_file(row):
    value1 = row[1]
    source2 = str(row[2])
    destination_folder = os.path.join("D:\\", row[3], row[4], row[0])

    if not os.path.exists(destination_folder):
        os.makedirs(destination_folder)

    save_path = os.path.join(destination_folder, value1)

    shutil.copy(source2, save_path)



with ThreadPoolExecutor(max_workers=8) as executor:
    # 提交任务到线程池
    futures = [executor.submit(copy_file, row) for row in data.itertuples(index=False)]

    # 等待所有任务完成
    for future in futures:
        future.result()

多线程这一步比较简单，主要不和数据库有交互，python封装得挺好的，比最开始快了七八倍