大数据测试---CSV文件分段读取上传

分段上传数据–CSV上传

def upload_stream(file, num=None):
    upload_url = “http://XXXXXXX”
    read_csv_file = pd.read_csv(file, iterator=True, low_memory=False)
    count = 0
    if num is None:
        cycles_numbers = 1000
    else:
        cycles_numbers = num // 1000 if num % 1000 == 0 else num // 1000 + 1
    for i in range(cycles_numbers):  # 设置循环次数
        try:
            user = read_csv_file.get_chunk(1000)  # 设置每次上传的条数
            count += user.shape[0]
            data = user.to_json(orient="records", force_ascii=False)
            d = requests.post(upload_url, data.encode(encoding='utf-8'), headers=get_header(), verify=False)
            print(d.text)
        except StopIteration:
            print("传完了:" + str(count))
            break
    print("上传数据行数:", count)
    print("上传数据列数:", user.shape[1])