# -*- coding:utf-8 -*-

'''
將tfidf寫入txt檔案
'''
import string
import sys
import codecs

reload(sys)
import os
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, TfidfTransformer

if __name__ == '__main__':

    # =====================1、構造樣本的目錄檔名稱======================

    # 根目錄檔案
 

    root_file = '.\Reduced_internet_preprocess'
    # 目錄列表
    text_dir_list = []
    # 遍歷根目錄檔案
    for root, dir, files in os.walk(root_file):
        # 構造檔案目錄
        for f in files:
            text_init_dir = root + '\\' + f
            text_dir_list.append(text_init_dir)

    # =================== 2、從檔案匯入停用詞表=======================
 

    stpwrdpath = "E:\ly\stop_words.txt"  # 停用詞檔案路徑
    stpwrd_dic = open(stpwrdpath, 'rb')
    stpwrd_content = stpwrd_dic.read()  # 將停用詞表轉換為list
    stpwrdlst = stpwrd_content.splitlines()
    stpwrd_dic.close()

    # ====================3、遍歷讀取樣本檔案內容，製作語料庫=========================
    # 儲存文件的語料庫
    corpus = []
    # 把內容寫入語料庫
 

    for f in text_dir_list:
        # 開啟文字檔案
        file_source = open(f, 'r+')
        content = file_source.read()
        file_source.close()
        corpus.append(content)

    # =====================4、對文件做詞向量化，計算每個詞語的tfidf值======================

    vectorizer = CountVectorizer(stop_words=stpwrdlst)
    transformer = TfidfTransformer()
    tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus))
    #vector = TfidfVectorizer(stop_words=stpwrdlst)  # 生成向量物件，input引數是檔案
    #tfidf = vector.fit_transform(corpus)

    word = vectorizer.get_feature_names()  # 所有文字的關鍵字
    weight = tfidf.toarray()  # 對應的tfidf矩陣

    print u'詞向量數:', len(word)
    print u'詞向量數:', len(weight[0])
    print u'詞向量'
    print word
    print u'樣本數：', len(weight)
    print u'權重矩陣'
    print weight

    # =======================5、把每個文件中詞語的tfidf值寫入檔案中儲存=====================
    # 建立每個樣本的tfidf值結果目錄
    sFilePath = '.\Tfidf_file'
    if not os.path.exists(sFilePath):
        os.mkdir(sFilePath)

    # 這裡將每份文件詞語的TF-IDF寫入tfidffile資料夾中儲存
    for i in range(len(weight)):
        print u"--------Writing all the tf-idf in the", i, u" file into ", sFilePath + '\\' + string.zfill(i,
                                                                                                          5) + '.dat', "--------"
        f = codecs.open(sFilePath + '\\' + string.zfill(i, 5) + '.dat', 'w', 'utf-8')
        # 寫入詞向量行
        for j in range(len(word)):
            f.write(word[j] + " ")
        f.write('\n')
        # 寫入tfidf
        for j in range(len(word)):
            f.write(str(weight[i][j]) + ' ')
        f.close()
    print '\n'
    # =======================6、合成一個文件====================================
    # 建立總結果目錄
    resultFilePath = '.\Result'
    if not os.path.exists(resultFilePath):
        os.mkdir(resultFilePath)
    # 把每個樣本文件的tfidf值寫入一個文件中
    f = codecs.open(resultFilePath + '\\' + 'alldata' + '.dat', 'w', 'utf-8')
    #for j in range(len(word)):
        #f.write(word[j] + " ")
    #f.write('\n')
    for i in range(len(weight)):
        print u"--------Writing all the tf-idf in the", i, u" file into ", resultFilePath + '\\' + 'weight_alldata' + '.dat', "--------"
        for j in range(len(word)):
            f.write(str(weight[i][j]) + ' ')
        f.write('\n')
    f.close()