将一个大文件通过流上传的接口,按行分割后进行上传
(这里用的文件是csv文件。其它文件的话,可以先转成csv文件;tqdm可以显示上传进度)
@func_time
def upload_stream(stream_id, file):
upload_url = baseURL + "/XXXXXX/{}/documents".format(stream_id)
user_info = pd.read_csv(file, iterator=True, low_memory=False)
count = 0
for i in tqdm(range(100)):
try:
user = user_info.get_chunk(1000)
count += user.shape[0]
data = user.to_json(orient="records", force_ascii=False)
requests.post(upload_url, data.encode(encoding='utf-8'), headers=get_header(), verify=False).json()
except StopIteration:
break
print("上传数据行数:", count)
print("上传数据列数:", user.shape[1])