python超大型数据集分割

直接上源码

import pandas as pd
import numpy as np
import glob,os
import openpyxl
path =r'001\信息.xlsx'   #文件路径
chunksize = 5000   #切分行数
i = 0
df = pd.read_excel(path,engine='openpyxl')
print('切分总数',len(np.array_split(df, len(df) // chunksize)))
for chunk in np.array_split(df, len(df) // chunksize):
    chunk.to_csv('001/file_{:02d}.csv'.format(i), index=False, header=True,encoding='gb18030')
    i += 1