Python 3.6 利用NLTK 統計多個文字中的詞頻
阿新 • • 發佈:2019-01-26
#!/usr/bin/env python
# encoding: utf-8
"""
@author: wg
@software: PyCharm
@file: word_frequency_statistics.py
@time: 2017/3/16 0016 10:46
"""
import os
import nltk
'''
利用NLTK 統計多個文字中的詞頻
'''
dirs = os.listdir('../../data/大秦帝國/') # 獲取根目錄
dictionary = {} # 空詞典,用於儲存最終的詞頻
stopwords = ['、','(',')',',','。',':','“' ,'”','\n\u3000','\u3000','的','‘','’'] # 停用詞
'''
def process():
for d in dirs: #遍歷根目錄下的資料夾
subdir = os.listdir('../../data/大秦帝國/')
for f in subdir: # 遍歷資料夾下的檔案
text = open('', 'r', encoding='utf-8').read() # 讀取文字內容
print('D:/sogouOutput/'+d+'/'+f)
fredist = nltk.FreqDist(text.split(' ')) # 獲取單檔案詞頻
for localkey in fredist.keys(): # 所有詞頻合併。 如果存在詞頻相加,否則新增
if localkey in stopwords: # 檢查是否為停用詞
print('-->停用詞:', localkey)
continue
if localkey in dictionary.keys(): # 檢查當前詞頻是否在字典中存在
dictionary[localkey] = dictionary[localkey] + fredist[localkey] # 如果存在,將詞頻累加,並更新字典值
print('--> 重複值:', localkey, dictionary[localkey])
else: # 如果字典中不存在
dictionary[localkey] = fredist[localkey] # 將當前詞頻新增到字典中
print('--> 新增值:', localkey, dictionary[localkey])
print('===================================================')
print(sorted(dictionary.items(), key = lambda x:x[1])) # 根據詞頻字典值排序,並列印
'''
def process():
subdir = os.listdir('../../data/wordcloud/')
for f in subdir: # 遍歷資料夾下的檔案
text = open('../../data/wordcloud/'+f, 'r', encoding='utf-8').read() # 讀取文字內容
print('../../data/wordcloud/'+f)
fredist = nltk.FreqDist(text.split(' ')) # 獲取單檔案詞頻
for localkey in fredist.keys(): # 所有詞頻合併。 如果存在詞頻相加,否則新增
if localkey in stopwords: # 檢查是否為停用詞
print('-->停用詞:', localkey)
continue
if localkey in dictionary.keys(): # 檢查當前詞頻是否在字典中存在
dictionary[localkey] = dictionary[localkey] + fredist[localkey] # 如果存在,將詞頻累加,並更新字典值
print('--> 重複值:', localkey, dictionary[localkey])
else: # 如果字典中不存在
dictionary[localkey] = fredist[localkey] # 將當前詞頻新增到字典中
print('--> 新增值:', localkey, dictionary[localkey])
print('===================================================')
print(sorted(dictionary.items(), key = lambda x:x[1])) # 根據詞頻字典值排序,並列印
if __name__ == '__main__':
process()