import nltk
nltk.download()
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

#預處理
def preprocessing(text):
    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokrnize(sent)]
    stops = stopwords.words('english')  
    tokens = [token for
 
 token in tokens if token not in stops]   #去掉停用詞

    tokens = [token.lower() for token in tokens if len(token)>=2]  #去掉長度小於2的詞
    lmtzr  =  WordNetLemmatizer()
    tokens = (lmtzr.lemmatize(token) for token in tokens) #詞性還原
    preprocessed_text = ' '.join(tokens)  
    return preprocessed_text

 
#讀取資料集
import csv
file_path = r'C:\Users\Administrator\Desktop\SMSSpamCollectionjsn.txt'
sms = open(file_path,'r',encoding='utf-8')
sms_data = []
sms_label = []
csv_reader = csv.reader(sms,delimiter = '\t')
for line in csv_reader:
    sms_label.append(line[0])
    sms_data.append(preprocessing(line[1]))
sms.close()

 
#訓練集和測試集資料劃分
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test = train_test_split(sms_data,sms_label,test_size = 0.3,random_state=0,stratify=sms_label)

#將其向量化
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(min_df=2,ngram_range=(1,2),stop_words='english',strip_accents='unicode',norm='12')
X_train = vectorizer.fit_transform(x_train)
X_test = vectorizer.transform(x_test)

#樸素貝葉斯分類器

from sklearn.navie_bayes import MultinomiaNB
clf = MultinomiaNB().fit(X_train,y_train)

#測試模型
y_nb_pred = clf.predict(X_test)

#測試模型：結果顯示
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

print(y_nb_pred.shape,y_nb_pred) #x_test預測結果
print('nb_confusion_matrix:')
cm = confusion_matrix(y_test,y_nb_pred)#混淆矩陣
print(cm)
print('nb_classification_report:')
cr = classification_report(y_test,y_nb_pred) #主要分類指標的文字報告
print(cr)