'''
min_df:的含義
min_df is used for removing terms that appear too infrequently. For example:
•min_df = 0.01 means "ignore terms that appear in less than 1% of the documents".
 
•min_df = 5 means "ignore terms that appear in less than 5 documents".
The default min_df is 1, which means "ignore terms that appear in less than 1 document".Thus, the default setting does not ignore any terms.
max_df:的含義
max_df is used for removing terms that appear too frequently, also known as "corpus-specific stop words". For example:
 
•max_df = 0.50 means "ignore terms that appear in more than 50% of the documents".
•max_df = 25 means "ignore terms that appear in more than 25 documents".
The default max_df is 1.0, which means "ignore terms that appear in more than 100% of the documents". Thus, the default setting does not ignore any terms.
 
'''

# coding=utf-8
mydoclist = [u'溫馨 提示 ： 家庭 暢享 套餐 介紹 、 主卡 新增 / 取消 副 卡 簡訊 辦理 方式 , 可 點選 文件 左上方  簡訊  圖示 即可 將 簡訊 指令 傳送給 客戶',
u'客戶 申請 i 我家 ， 家庭 暢享 計劃  後 ， 可 選擇 設定 1 - 6 個 同一 歸屬 地 的 中國移動 網 內 號碼 作為 親情 號碼 ， 組建 一個 家庭 親情 網  家庭 內 ',
u'所有 成員 可 享受 本地 互打 免費 優惠 ， 家庭 主卡 號碼 還 可 享受 省內 / 國內 漫遊 接聽 免費 的 優惠']
from sklearn.feature_extraction.text import CountVectorizer

# count_vectorizer = CountVectorizer(min_df=1)
# term_freq_matrix = count_vectorizer.fit_transform(mydoclist)
# print "Vocabulary:", count_vectorizer.vocabulary_
#
# from sklearn.feature_extraction.text import TfidfTransformer
#
# tfidf = TfidfTransformer(norm="l2")
# tfidf.fit(term_freq_matrix)
#
# tf_idf_matrix = tfidf.transform(term_freq_matrix)
# print tf_idf_matrix.todense()
# from __future__ import print_function


from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer(min_df = 1)
tfidf_matrix = tfidf_vectorizer.fit_transform(mydoclist)
str=''
for i in tfidf_vectorizer.vocabulary_:
    str+=' '+i
print str
print tfidf_matrix.todense()
new_docs = [u'一個 客戶 號碼 只能 辦理 一種 家庭 暢享 計劃 套餐 ， 且 只能 加入 一個 家庭網']
new_term_freq_matrix = tfidf_vectorizer.transform(new_docs)
print tfidf_vectorizer.vocabulary_,type(tfidf_vectorizer.vocabulary_)
str=''
for i,j in sorted(tfidf_vectorizer.vocabulary_.items(), key=lambda d: d[1]):
    str+=' '+i
print str
print [ v for v in sorted(tfidf_vectorizer.vocabulary_.values())]
print sorted(tfidf_vectorizer.vocabulary_.items(), key=lambda d: d[1])


print new_term_freq_matrix.todense()