python pandas使用chunksize异步拆分固定行数的文件

添加链接描述

import pandas as pd
import asyncio
from collections import defaultdict
 
 
collect = defaultdict(list)
 
#######创建处理一个对象的方法，并返回期物
async def dealone(chunk,sas):
    path='/data/chaifen/testphone_%d.txt'%sas
    chunk.to_csv(path,index=False)
    return path
 
async def submitor(chunk,sas):
    future = await dealone(chunk,sas)
    collect['res'].append(future)
    #print(future)
 
 
chunks = pd.read_csv('/data/testphone.txt',dtype='object',chunksize=3)
 
loop = asyncio.get_event_loop()
tasks = asyncio.wait([submitor(chunk,index) for index,chunk in enumerate(chunks,1)])
loop.run_until_complete(tasks)
print(collect['res'])