pandas 讀取大檔案
阿新 • • 發佈:2019-02-20
from matplotlib import pyplot
import scipy as sp
import numpy as np
from matplotlib import pylab
from pandas import Series, DataFrame
import pandas as pd
filename = 'data.txt'
data = pd.read_csv(filename, sep='::', names=list('uirt'), index_col=False, usecols=range(len('uirt')),engine = 'python' , iterator=True)
loop = True
chunkSize = 100000
chunks = []
while loop:
try:
chunk = data.get_chunk(chunkSize)
chunks.append(chunk)
except StopIteration:
loop = False
print "Iteration is stopped."
data = pd.concat(chunks, ignore_index=True)