1、sklearn包計算
1.1 transformer函式計算

from sklearn.feature_extraction.text import TfidfTransformer
if __name__ == "__main__":
    corpus=["我 來到 北京 清華大學",
    "小明 碩士 畢業 與 中國 科學院",
    "我 愛 北京 天安門"]
    vectorizer=CountVectorizer()
    transformer=TfidfTransformer()
tfidf=transformer.fit_transform(vectorizer.fit
 
_transform(corpus))
    word=vectorizer.get_feature_names()
    weight=tfidf.toarray()

1.2TfidfVectorizer函式計算

from sklearn.feature_extraction.text import TfidfVectorizer
count_vec = TfidfVectorizer()  
x_train = count_vec.fit_transform(sentences_train)  
x_test  = count_vec.transform(sentences_test)  
 

TfidfVectorizer函式的輸入為分詞後的句子列表，而transformer函式需要先用CountVectorizer函式把句子列表變為詞袋模型在轉換為tfidf值

2、gensim包計算tfidf值

walk = os.walk('/u01/jerry/Reduced')
for root, dirs, files in walk:
    for name in files:
        f = open(os.path.join(root, name), 'r')
   raw = f.read()
   word_list = list(jieba.cut(raw, cut_all = False))
   train_set.append(word_list)


dic = corpora.Dictionary(train_set)
corpus = [dic.doc2bow(text
 
) for text in train_set]
tfidf = models.TfidfModel(corpus)
corpus_tfidf = tfidf[corpus]
lda = models.LdaModel(corpus_tfidf, id2word = dic, num_topics = 10)
corpus_lda = lda[corpus_tfidf]