理论参考:朴素贝叶斯理论推导与三种常见模型
import jieba
import jieba.posseg as pseg
import json
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB,BernoulliNB,GaussianNB
from sklearn.model_selection import train_test_split
from sklearn import metrics
def StopWordsList():
filepath = 'E:\数据挖掘学习\FAQ\stopwords.txt'
wlst = [w.strip() for w in open(filepath, 'r', encoding='utf8').readlines()]
return wlst
def seg_sentence(sentence, stop_words):
stop_flag = ['x', 'c', 'u', 'd', 'p', 't', 'uj', 'm', 'f', 'r'] # 过滤数字m
sentence_seged = pseg.cut(re.sub('\n', '', sentence))
outstr = []
for word, flag in sentence_seged:
if word not in stop_words and flag not in stop_flag:
outstr.append(word)
return ' '.join(outstr)
def model(tfidf,y):
# 构建朴素贝叶斯分类器0.959
# clf = MultinomialNB(alpha=0.1).fit(tfidf,y)
clf = BernoulliNB(alpha=0.1).fit(tfidf,y)
# clf = GaussianNB(alpha=0.1).fit(tfidf,y)
return clf
# 加载数据
stop_words = StopWordsList()
path = 'E:\python\spider\差评.txt'
file = open(path, 'r', encoding='utf8')
# negative = [seg_sentence(s, stop_words) for s in file]
negative = []
for s in file:
if '此用户未填写评价内容' not in s:
negative.append(seg_sentence(s, stop_words))
file.close()
path = 'E:\python\spider\好评.txt'
file = open(path, 'r', encoding='utf8')
# positive = [seg_sentence(s, stop_words) for s in file]
positive = []
for s in file:
if '此用户未填写评价内容' not in s:
positive.append(seg_sentence(s, stop_words))
file.close()
# print(positive)
corpus = negative + positive
y = [0] * len(negative) + [1] * len(positive)
x_train, x_test, y_train, y_test = train_test_split(corpus, y, test_size=0.3)
vector = CountVectorizer()
# vector = TfidfVectorizer()
# 训练集先拟合数据,再标准化
xtrain = vector.fit_transform(x_train)
# 数据标准化
xtest = vector.transform(x_test)
#模型训练
clf = model(xtrain, y_train)
pre = clf.predict(xtrain)
# 评估
print('训练数据准确率:',metrics.f1_score(pre, y_train, average='micro'))
pre = clf.predict(xtest)
print('测试数据准确率:',metrics.f1_score(pre, y_test, average='micro'))