python pandas使用chunksize異步拆分固定行數的文件
阿新 • • 發佈:2019-01-14
csv one als asyncio return test loop false lse
import pandas as pd import asyncio from collections import defaultdict collect = defaultdict(list) #######創建處理一個對象的方法,並返回期物 async def dealone(chunk,sas): path=‘/data/chaifen/testphone_%d.txt‘%sas chunk.to_csv(path,index=False) return path async def submitor(chunk,sas): future = await dealone(chunk,sas) collect[‘res‘].append(future) #print(future) chunks = pd.read_csv(‘/data/testphone.txt‘,dtype=‘object‘,chunksize=3) loop = asyncio.get_event_loop() tasks = asyncio.wait([submitor(chunk,index) for index,chunk in enumerate(chunks,1)]) loop.run_until_complete(tasks) print(collect[‘res‘])
python pandas使用chunksize異步拆分固定行數的文件