text = ‘"Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat..."‘

import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
nltk.download()

#預處理
def preprocessing(text):
#text = text.decode("utf-8)
tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
stops = stopwords.words(‘english‘)
tokens = [token for token in tokens if token not in stops]

tokens = [token.lower() for token in tokens if len(token)>=3]
lmtzr = WordNetLemmatizer()
tokens = [lmtzr.lenmatize(token) for token in tokens]
preprocessed_text = ‘ ‘.join(tokens)
return preprocessed_text

preprocessing(text)

import csv #用csv讀取郵件數據，分解出郵件類別及郵件內容
file_path = r‘C:\Users\Administrator\Desktop\SMSSpamCollectionjsn.txt‘
sms = open(file_path,‘r‘,encoding = ‘utf-8‘)
sms_data = []
sms_label = []
csv_reader = csv.reader(sms,delimiter=‘\t‘)
for line in csv_reader:
sms_label.append(line[0])
sms_data.append(processing[1])
sms.close()
sms_label
sms_data

from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test = train_test_split(sms_data,sms_label,test_size=0.3,random_state=0,stratify=sms_label) #訓練集，測試集

#將其向量化
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(min_df = 2,ngram_range=(1,2),stop_words=‘english‘,strip_accents=‘unicode‘,norm=‘l2‘)
x_train = vectorizer.fit_transform(x_train)
x_test = vectorizer.transform(x_test)

#樸素貝葉斯分類器
from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB().fit(x_train,y_train)
y_nb_pred = clf.predict(x_test)

#分類結果顯示
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

print(y_nb_pred.shape,y_nb_pred) #x_text預測結果
print(‘nb_confusion_matrix:‘)
cm = confusion.matrix(y_test,y_nb_pred) #混淆矩陣
print(cm)
print(‘nb_classification_report‘)
cr = classification_report(y_test,y_nb_pred) #主要分類指標的文本報告
print(cr)

feature_name = vectorizer.get_feature_name() #出現過的單詞列表
coefs = clf.coef_ #先驗證概率
intercept = clf.intercept_
coef_with_fns = sorted(zip(coefs[0],feature_names)) #對數概率p(x_i)y與單詞x_i映射

n=10
top = zip(coefs_with_fns[:n],coefs_with_fns[:(n+1):-1])
for(coef_1,fn_1),(coef_2,fn_2) in top:
print(‘‘)

text=‘"As per your request Melle Melle Oru Minnaminunginte Nurungu Vettam has been set as your callertune for all Callers. Press *9 to copy your friends Callertune"‘

import nltk #nltk進行分詞
for sent in nltk.sent_tokenize(text): #對文本按照句子進行分割
for word in nltk.word_tokenize(sent): #對句子進行分詞
print(word)
tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]

from nltk.corpus import stopwords #去掉停用詞
stops = stopwords.words(‘english‘)
stops

tokens = [token for token in tokens if token not in stops]
s = set(tokens)-set(stops)
print(len(tokens),len(set(tokens)),len(s))

# nltk.download(‘wordnet‘)
from nltk.stem import WordNetLemmatizer #詞性還原
lemmatizer = WordNetLemmatizer()
lemmatizer.lemmatize(‘leavers‘)

11.29作業