在一些比賽中，經常會出現原始訓練資料就有十幾G大小，正常的個人電腦記憶體根本不足以容納這麼大資料量。查到可以使用Pandas將原資料集劃分成小塊儲存。以下內容轉載自知乎。

user_feat = ['user_id','user_gender_id','user_age_level','user_occupation_id','user_star_level']
reader = pd.read_csv("./data/round2_train.txt", sep="\s+",iterator=True)
chunks = []
loop = True
while loop:
    try
 
:
        chunk = reader.get_chunk(500000)[user_feat]
        chunks.append(chunk)
    except StopIteration:
        loop = False
        print("Iteration is stopped")
df_user = pd.concat(chunks,axis=0, ignore_index=True)
df_user = pd.concat([df_user, test[user_feat]],axis=0)
df_user.drop_duplicates(subset='user_id'
 
,keep='first',inplace=True)
df_user.to_csv('./data/user_file.csv',index=False)
print('user_file', df_user.shape)

del df_user
gc.collect()