1. 使用朴素贝叶斯过滤垃圾邮件
文件解析及完整的垃圾邮件测试函数:
# 文件解析及完整的垃圾邮件测试函数
def textParse(bigString): # input is big string, #output is word list
import re
listOfTokens = re.split(r'\W*', bigString)
return [tok.lower() for tok in listOfTokens if len(tok) > 2]
def spamTest():
docList=[]; classList = []; fullText =[]
# 导入文件夹spam和ham下的文本文件,并将它们解析为词列表。
for i in range(1,26):
wordList = textParse(open('email/spam/%d.txt' % i).read())
docList.append(wordList) # 在列表末尾添加新的对象 [...[]]
fullText.extend(wordList) # 在列表末尾添加新的序列并与原序列合并 [....]
classList.append(1)
wordList = textParse(open('email/ham/%d.txt' % i).read())
docList.append(wordList)
fullText.extend(wordList)
classList.append(0)
# 创建包含文档中所有词的不重复列表
vocabList = createVocabList(docList)#create vocabulary
trainingSet = list(range(50)); testSet=[] #create test set
# 随机抽取10封电子邮件作为测试集
for i in range(10):
# random.uniform随机生成一个数字在(x,y)中
randIndex = int(random.uniform(0,len(trainingSet)))
testSet.append(trainingSet[randIndex])
# 选择出的数字对应的文档被添加到测试集中,同时也从训练集中删除
del(trainingSet[randIndex])
trainMat=[]; trainClasses = []
# 遍历全部训练集,对每个文档构建词向量。
for docIndex in trainingSet:#train the classifier (get probs) trainNB0
trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
trainClasses.append(classList[docIndex])
p0V,p1V,pSpam = trainNB0(array(trainMat),array(trainClasses))
errorCount = 0
for docIndex in testSet: #classify the remaining items
wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
if classifyNB(array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:
errorCount += 1
print("classification error",docList[docIndex])
print('the error rate is: ',float(errorCount)/len(testSet))
#return vocabList,fullText
对上述的程序进行尝试:
这里一直出现的错误是将垃圾邮件误判为正常邮件,相比之下,将正常邮件误判为垃圾邮件会比将垃圾邮件误判为这正常邮件要差。为了避免错误,我们将会在后面介绍其他方式修正分类器。