目录
1. 准备数据:从文本中构建词向量
1.1 词表到向量的转换函数
def loaddataset(): # 创建一些实验样本
postinglist = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
['stop', 'posting', 'stupid', 'worthless', 'garbage'],
['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
classvec = [0, 1, 0, 1, 0, 1] # 1代表侮辱性文字,0代表正常言论
return postinglist, classvec
# postinglist:进行词条切分后的文档集合
# classvec:类别标签的集合
def createvocablist(dataset): # 会创建一个包含在所有文档中出现的不重复词的列表
vocabset = set([]) # 创建一个空集
for document in dataset:
vocabset = vocabset | set(document) # 创建两个集合的并集
return list(vocabset)
def setofwords2vec(vocablist, inputset):
# 输入参数为词汇表及某个文档
returnvec = [0] * len(vocablist) # 创建一个和词汇表等长的向量,并将其元素都设为0
for word in inputset: # 遍历文档中的所有单词
if word in vocablist:
returnvec[vocablist.index(word)] = 1
# 如果出现了词汇表中的单词,则将输出的文档向量中的对应值设为1
else:
print("the word: %s is not in my Vocabulary!" % word)
return returnvec
# 输出的是文档向量,向量的每一元素为1或0
# 分别表示词汇表中的单词在输入文档中是否出现
查看函数执行效果:
listOposts, listclasses = loaddataset()
myvocablist = createvocablist(listOposts)
print(myvocablist)
输出结果:
查看setofwords2vec()的运行结果:
print(setofwords2vec(myvocablist, listOposts[0]))
print(setofwords2vec(myvocablist, listOposts[3]))
输出结果:
2. 训练算法:从词向量计算概率
import numpy as np
def trainnb0(trainmatrix,traincategory):
# trainmatrix: 文档矩阵;
# traincategory:由每篇文档类别标签所构成的向量
numtraindocs = len(trainmatrix)
numwords = len(trainmatrix[0])
pabusive = sum(traincategory)/float(numtraindocs)
"下两行初始化分子变量和分母变量"
p0num = np.zeros(numwords); p1num = np.zeros(numwords)
p0denom = 0.0; p1denom = 0.0
for i in range(numtraindocs):
if traincategory[i] == 1:
p1num += trainmatrix[i]
p1denom += sum(trainmatrix[i])
else:
p0num += trainmatrix[i]
p0denom += sum(trainmatrix[i])
p1vect = p1num/p1denom
p0vect = p0num/p0denom
return p0vect,p1vect,pabusive
从预先加载值中调入数据:
listOposts, listclasses = loaddataset()
构建一个包含所有词的列表myvocablist:
myvocablist = createvocablist(listOposts)
for循环使用词向量来填充trainmat列表:
trainmat = []
for postindoc in listOposts:
trainmat.append(setofwords2vec(myvocablist, postindoc))
下面给出属于侮辱性文档的概率以及两个类别的概率向量:
p0v, p1v, pab = trainnb0(trainmat, listclasses)
print("pab:", pab)
print("属于侮辱性文档的概率:\n", p0v)
print("属于侮辱性文档的概率:\n", p1v)
输出结果:
3. 测试算法:根据现实情况修改分类器
修改后的trainb0()函数:
def trainnb0(trainmatrix, traincategory):
# trainmatrix: 文档矩阵;
# traincategory:由每篇文档类别标签所构成的向量
numtraindocs = len(trainmatrix)
numwords = len(trainmatrix[0])
pabusive = sum(traincategory)/float(numtraindocs)
"下两行初始化分子变量和分母变量"
p0num = np.ones(numwords); p1num = np.ones(numwords)
p0denom = 2.0; p1denom = 2.0
for i in range(numtraindocs):
if traincategory[i] == 1:
p1num += trainmatrix[i]
p1denom += sum(trainmatrix[i])
else:
p0num += trainmatrix[i]
p0denom += sum(trainmatrix[i])
p1vect = np.log(p1num/p1denom)
p0vect = np.log(p0num/p0denom)
return p0vect,p1vect,pabusive
3.1 朴素贝叶斯分类函数
def classifynb(vec2classify, p0vec, p1vec, pclass1):
p1 = sum(vec2classify * p1vec) + np.log(pclass1) # element-wise mult
p0 = sum(vec2classify * p0vec) + np.log(1.0 - pclass1)
if p1 > p0:
return 1
else:
return 0
def testingnb():
listOposts, listclasses = loaddataset()
myvocablist = createvocablist(listOposts)
trainmat = []
for postindoc in listOposts:
trainmat.append(setofwords2vec(myvocablist, postindoc))
p0v, p1v, pab = trainnb0(np.array(trainmat), np.array(listclasses))
testentry = ['love', 'my', 'dalmation']
thisdoc = np.array(setofwords2vec(myvocablist, testentry))
print(testentry, 'classified as: ', classifynb(thisdoc, p0v, p1v, pab))
testentry = ['stupid', 'garbage']
thisdoc = np.array(setofwords2vec(myvocablist, testentry))
print(testentry, 'classified as: ', classifynb(thisdoc, p0v, p1v, pab))
查看分类器输出结果:
testingnb()
输出结果:
4. 准备数据: 文档词袋模型
def bagofwords2Vecmn(vocablist, inputset):
returnVec = [0] * len(vocablist)
for word in inputset:
if word in vocablist:
returnVec[vocablist.index(word)] += 1
return returnVec