python情感分析算法的时间复杂度_Python贝叶斯算法进行情感分析

from __future__ import division

import re

from numpy import ones, array

from numpy.lib.scimath import log

from nltk import *

def loadDataSet():

pos=open("pos_train.txt",'r')

neg=open("neg_train.txt",'r')

lst_all=[]

classVec=[]

for i in range(700):

classVec.append(i%2)

for i in range(350):

str0=pos.readline()

str1=neg.readline()

regEx0=re.compile('\\W*')

regEx1=re.compile('\\W*')

lst_pos=regEx0.split(str0)

lst_neg=regEx1.split(str1)

lst_all.append([tok.lower() for tok in lst_pos if len(tok)>0])

lst_all.append([tok.lower() for tok in lst_neg if len(tok)>0])

return lst_all,classVec

def loadTestSet():

pos=open("pos_test.txt",'r')

neg=open("neg_test.txt",'r')

lst_pos_test=[]

lst_neg_test=[]

for i in range(350):

str0=pos.readline()

regEx0=re.compile('\\W*')

lst_pos=regEx0.split(str0)

lst_pos_test.append([tok.lower() for tok in lst_pos if len(tok)>0])

for i in range(350):

str1=neg.readline()

regEx1=re.compile('\\W*')

lst_neg=regEx1.split(str1)

lst_neg_test.append([tok.lower() for tok in lst_neg if len(tok)>0])

# print 'loadtestset'

return lst_pos_test,lst_neg_test

def createVocabList(dataSet):

vocabSet = set([]) #create empty set

for document in dataSet:

vocabSet = vocabSet | set(document) #union of the two sets

# print "createVocabList"

return list(vocabSet)

def bagOfWords2VecMN(vocabList, inputSet):

returnVec = [0]*len(vocabList)

for word in inputSet:

if word in vocabList:

returnVec[vocabList.index(word)] += 1

# else:

# print "the word: %s is not in my Vocabulary!" % word

# print "bagbagbag"

return returnVec

def trainNB0(trainMatrix,trainCategory):

numTrainDocs = len(trainMatrix)

numWords = len(trainMatrix[0])

pCi = sum(trainCategory)/float(numTrainDocs)

p0Num = ones(numWords); p1Num = ones(numWords) #change to ones()

p0Denom = 2.0; p1Denom = 2.0 #change to 2.0

for i in range(numTrainDocs):

if trainCategory[i] == 1:

p1Num += trainMatrix[i]

p1Denom += sum(trainMatrix[i])

else:

p0Num += trainMatrix[i]

p0Denom += sum(trainMatrix[i])

p1Vect = log(p1Num/p1Denom) #change to log()

p0Vect = log(p0Num/p0Denom) #change to log()

print "training"

return p0Vect,p1Vect,pCi

def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):

p1 = sum(vec2Classify * p1Vec) + log(pClass1) #element-wise mult

p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)

# print "classifying"

if p1 > p0:

return 1

else:

return 0

def testingNB(lst_pos,lst_neg):

listOPosts,listClasses = loadDataSet()

myVocabList = createVocabList(listOPosts)

trainMat=[]

pos_corre=[]

neg_corre=[]

for postinDoc in listOPosts:

trainMat.append(bagOfWords2VecMN(myVocabList, postinDoc))

p0V,p1V,pAb = trainNB0(array(trainMat),array(listClasses))

for i in range(350):

testEntry=lst_pos[i]

thisDoc = array(bagOfWords2VecMN(myVocabList, testEntry))

a=classifyNB(thisDoc,p0V,p1V,pAb)

pos_corre.append(a)

print("the positive text classify accuracy: {} ".format(1-sum(pos_corre)/350))

print(sum(pos_corre))

for i in range(350):

testEntry = lst_neg[i]

thisDoc = array(bagOfWords2VecMN(myVocabList, testEntry))

a=classifyNB(thisDoc,p0V,p1V,pAb)

neg_corre.append(a)

print("the negative text classify accuracy: {} ".format(sum(neg_corre)/350))

print(sum(neg_corre))

print(p0V)

print(p1V)

print(pAb)

if __name__=='__main__':

lst_pos,lst_neg=loadTestSet()

testingNB(lst_pos,lst_neg)