基本步骤(2分类):
根据要分类的文本,建立不重复词的列表->对于每一条文本,转化‘文档向量’(此向量长度为上一步的列表,值为1或者0)
->根据计算分类概率,大概思路是训练一个文档集合(已转化为文档向量),若文档为1,把文档list对应位置相加作为分子,同时分母+1,这样最后加起来的list再除分母(后面称这个分子除分母为listA)就得到了文档为1时所有单词的概率,即
,在预测x文档向量时,直接可以用x与listA对应位置相乘,得到的就是x的
,因为x也是文档向量,对应位置相乘后不是x的
就已经为0了,最后再把这个x的
list求和就行了,为什么不相乘,因为已经转化为log了,见下面代码。文档为0时同理,比较下哪个大就行了
具体代码:
import numpy as np
#创建数据集
def loadDataSet():
postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
['stop', 'posting', 'stupid', 'worthless', 'garbage'],
['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
classVec = [0, 1, 0, 1, 0, 1] # 1 is abusive, 0 not
return postingList, classVec
#将数据集的所有条记录合并为一个集合
def createVocabList(dataSet):
vocabSet = set([]) # create empty set
for document in dataSet:
vocabSet = vocabSet | set(document) # union of the two sets
return list(vocabSet)
#判断inputSet中的单词是否出现在vocabList中,并返回位置
def setOfWords2Vec(vocabList, inputSet):
returnVec = [0] * len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:
print("the word: %s is not in my Vocabulary!" % word)
return returnVec
#算出每一个单词在其分类中的条件概率(某类别单词在此类别所有文章中出现的次数÷此类别所有的单词数),得出的是最能代表此类别的单词
def trainNB0(trainMatrix,trainCategory):
numTrainDocs = len(trainMatrix)
numWords = len(trainMatrix[0])
pAbusive = sum(trainCategory)/float(numTrainDocs) #垃圾邮件的比例(侮辱性词汇的概率,即p(c))
p0Num = np.ones(numWords); p1Num = np.ones(numWords) #change to ones()
p0Denom = 2.0; p1Denom = 2.0 #change to 2.0
for i in range(numTrainDocs):
if trainCategory[i] == 1: #求p(a|c=1)
p1Num += trainMatrix[i]
p1Denom =p1Denom+1
else: #求p(a|c=0)
p0Num += trainMatrix[i]
p0Denom =p0Denom+1
p1Vect = np.log(p1Num/p1Denom) #change to log() 转成log后可将本来的连乘变为连加,防止概率过小相乘后为0
p0Vect = np.log(p0Num/p0Denom) #change to log()
return p0Vect,p1Vect,pAbusive
# listOPosts,listClasses=loadDataSet()
# myVocabList=createVocabList(listOPosts)
# print("myVoc:",myVocabList)
# trainMat=[]
# for postinDoc in listOPosts:
# trainMat.append(setOfWords2Vec(myVocabList,postinDoc))
# p0v,p1v,pAb=trainNB0(trainMat,listClasses)
# print('p1v',p1v)
#分类方法
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
p1 = sum(vec2Classify * p1Vec) + np.log(pClass1) #element-wise mult
p0 = sum(vec2Classify * p0Vec) + np.log(1.0 - pClass1)
if p1 > p0:
return 1
else:
return 0
def bagOfWords2VecMN(vocabList, inputSet):
returnVec = [0] * len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] += 1
return returnVec
#简单文本分类测试
def testingNB():
listOPosts, listClasses = loadDataSet()
myVocabList = createVocabList(listOPosts)
trainMat = []
for postinDoc in listOPosts:
trainMat.append(setOfWords2Vec(myVocabList, postinDoc))
p0V, p1V, pAb = trainNB0(np.array(trainMat), np.array(listClasses))
testEntry = ['love', 'my', 'dalmation']
thisDoc = np.array(setOfWords2Vec(myVocabList, testEntry))
print(testEntry, 'classified as: ', classifyNB(thisDoc, p0V, p1V, pAb))
testEntry = ['stupid', 'garbage']
thisDoc = np.array(setOfWords2Vec(myVocabList, testEntry))
print(testEntry, 'classified as: ', classifyNB(thisDoc, p0V, p1V, pAb))
# testingNB()