python+sklearn+基于贝叶斯实现商品评论情感分类

理论参考:朴素贝叶斯理论推导与三种常见模型

import jieba
import jieba.posseg as pseg
import json
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB,BernoulliNB,GaussianNB
from sklearn.model_selection import train_test_split
from sklearn import metrics

def StopWordsList():
    filepath = 'E:\数据挖掘学习\FAQ\stopwords.txt'
    wlst = [w.strip() for w in open(filepath, 'r', encoding='utf8').readlines()]
    return wlst

def seg_sentence(sentence, stop_words):
    stop_flag = ['x', 'c', 'u', 'd', 'p', 't', 'uj', 'm', 'f', 'r']  # 过滤数字m
    sentence_seged = pseg.cut(re.sub('\n', '', sentence))
    outstr = []
    for word, flag in sentence_seged:
        if word not in stop_words and flag not in stop_flag:
            outstr.append(word)
    return ' '.join(outstr)

def model(tfidf,y):
    # 构建朴素贝叶斯分类器0.959
#     clf = MultinomialNB(alpha=0.1).fit(tfidf,y)
    clf = BernoulliNB(alpha=0.1).fit(tfidf,y)
#     clf = GaussianNB(alpha=0.1).fit(tfidf,y)
    return clf


# 加载数据
stop_words = StopWordsList()
path = 'E:\python\spider\差评.txt'
file = open(path, 'r', encoding='utf8')
# negative = [seg_sentence(s, stop_words) for s in file]
negative = []
for s in file:
    if '此用户未填写评价内容' not in s:
        negative.append(seg_sentence(s, stop_words))
file.close()
path = 'E:\python\spider\好评.txt'
file = open(path, 'r', encoding='utf8')
# positive = [seg_sentence(s, stop_words) for s in file]
positive = []
for s in file:
    if '此用户未填写评价内容' not in s:
        positive.append(seg_sentence(s, stop_words))

file.close()
# print(positive)
corpus = negative + positive
y = [0] * len(negative) + [1] * len(positive)

x_train, x_test, y_train, y_test = train_test_split(corpus, y, test_size=0.3)

vector = CountVectorizer()
# vector = TfidfVectorizer()
# 训练集先拟合数据,再标准化
xtrain = vector.fit_transform(x_train)
# 数据标准化
xtest = vector.transform(x_test)

#模型训练
clf = model(xtrain, y_train)
pre = clf.predict(xtrain)

# 评估
print('训练数据准确率:',metrics.f1_score(pre, y_train, average='micro'))
pre = clf.predict(xtest)
print('测试数据准确率:',metrics.f1_score(pre, y_test, average='micro'))
  • 0
    点赞
  • 10
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值