########資料匯入

def review_to_wordlist(review):
”’
把IMDB的評論轉成詞序列
”’
# 去掉HTML標籤，拿到內容
review=BeautifulSoup(review,”html5lib”)
review_text = review.get_text()
# 用正則表示式取出符合規範的部分
review_text = re.sub(“[^a-zA-Z]”,” “, review_text)
# 小寫化所有的詞，並轉成詞list
words = review_text.lower().split()
# 返回words
return words

使用pandas讀入訓練和測試csv檔案

train = pd.read_csv(‘F://python3.5//Machine Learning//tensorflow//bynet//Bags_of_Popcorn//Kaggle//labeledTrainData//labeledTrainData.tsv’, header=0, delimiter=”\t”, quoting=3)
test = pd.read_csv(‘F://python3.5//Machine Learning//tensorflow//bynet//Bags_of_Popcorn//Kaggle//testData//testData.tsv’, header=0, delimiter=”\t”, quoting=3 )

print(train.review)

取出情感標籤，positive/褒 或者 negative/貶

y_train = train[‘sentiment’]

print(y_train[:32])

將訓練和測試資料都轉成詞list

train_data = []
for i in range(0,len(train[‘review’])):
train_data.append(” “.join(review_to_wordlist(train[‘review’][i])))
test_data = []
for i in range(0,len(test[‘review’])):
test_data.append(” “.join(review_to_wordlist(test[‘review’][i])))

特徵處理

from sklearn.feature_extraction.text import TfidfVectorizer as TFIV

初始化TFIV物件，去停用詞，加2元語言模型

tfv = TFIV(min_df=3, max_features=None, strip_accents=’unicode’, analyzer=’word’,token_pattern=r’\w{1,}’, ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1, stop_words = ‘english’)

合併訓練和測試集以便進行TFIDF向量化操作

X_all = train_data + test_data
len_train = len(train_data)

這一步有點慢，去喝杯茶刷會兒微博知乎歇會兒…

tfv.fit(X_all)
X_all = tfv.transform(X_all)

恢復成訓練集和測試集部分

X = X_all[:len_train]
X_test = X_all[len_train:]

##########################建模

多項式樸素貝葉斯

from sklearn.naive_bayes import MultinomialNB as MNB

model_NB = MNB()
model_NB.fit(X, y_train) #特徵資料直接灌進來
MNB(alpha=1.0, class_prior=None, fit_prior=True)

from sklearn.cross_validation import cross_val_score
import numpy as np

print(“多項式貝葉斯分類器20折交叉驗證得分: “, np.mean(cross_val_score(model_NB, X, y_train, cv=20, scoring=’roc_auc’)))