朴素bayes实战

from numpy import *


###创建一些实验样本#####################
def loadDataSet():
    postingList=[['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                 ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                 ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                 ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                 ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                 ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
    classVec = [0,1,0,1,0,1]    #1 is abusive, 0 not
    return postingList,classVec

####创建一个不重复的词表#############               
def createVocabList(dataSet):
    vocabSet = set([])  #创建空集合
    for document in dataSet:
        vocabSet = vocabSet | set(document) # 操作符|用于求两个集合的并集
    return list(vocabSet)



"""该函数的输入为词汇表及某个文档，输出为文档向量"""
def setOfWords2Vec(vocabList, inputSet):
    returnVec = [0]*len(vocabList)   # 创建一个其中所含元素都为0的向量
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] = 1   ######list.index(obj)返回查找对象的索引位置
        else: 
            print("the word: %s is not in my Vocabulary!" % word)
    return returnVec



def trainNB0(trainMatrix,trainCategory):
    numTrainDocs = len(trainMatrix)   # 计算矩阵行数
    numWords = len(trainMatrix[0])    # 计算矩阵列数
    pAbusive = sum(trainCategory)/float(numTrainDocs)  # 计算各个类别的概率
    p0Num = ones(numWords); p1Num = ones(numWords)      #change to ones() 
    p0Denom = 2.0; p1Denom = 2.0                        #change to 2.0
    '''接下来计算词汇表中各个词汇在不同分类中出现的概率'''
    for i in range(numTrainDocs):    # 依次读取文件
        if trainCategory[i] == 1:    #  这里的if函数判断文档类别
            p1Num += trainMatrix[i]    #  判断词在文档中出现的个数
            p1Denom += sum(trainMatrix[i])  #判断某个文档中的次总数
        else:
            p0Num += trainMatrix[i]
            p0Denom += sum(trainMatrix[i])
    p1Vect = log(p1Num/p1Denom)          #change to log()
    p0Vect = log(p0Num/p0Denom)          #change to log()
    return p0Vect,p1Vect,pAbusive



###朴素bayes分类函数##########
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
    p1 = sum(vec2Classify * p1Vec) + log(pClass1)    #element-wise mult
    p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
    if p1 > p0:
        return 1
    else: 
        return 0



########bayes词袋模型####################################
def bagOfWords2VecMN(vocabList, inputSet):
    returnVec = [0]*len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] += 1
    return returnVec

def testingNB():
    listOPosts,listClasses = loadDataSet()
    myVocabList = createVocabList(listOPosts)
    trainMat=[]
    for postinDoc in listOPosts:
        trainMat.append(setOfWords2Vec(myVocabList, postinDoc))
    p0V,p1V,pAb = trainNB0(array(trainMat),array(listClasses))
    testEntry = ['love', 'my', 'dalmation']
    thisDoc = array(setOfWords2Vec(myVocabList, testEntry))
    print(testEntry,'classified as: ',classifyNB(thisDoc,p0V,p1V,pAb))
    testEntry = ['stupid', 'garbage']
    thisDoc = array(setOfWords2Vec(myVocabList, testEntry))
    print(testEntry,'classified as: ',classifyNB(thisDoc,p0V,p1V,pAb))


# print(testingNB())

listOPosts,listClass=loadDataSet()

myVocabList=createVocabList(listOPosts)


trainMat=[]
for postinDoc in listOPosts:
    trainMat.append(setOfWords2Vec(myVocabList, postinDoc))


pOV,p1V,pAb=trainNB0(trainMat,listClass)
print(p1V)





# ##################利用bayes过滤垃圾邮件################



# ####文件解析###############
# def textParse(bigString):    #input is big string, #output is word list
#     import re
#     listOfTokens = re.split(r'\W*', bigString)
#     return [tok.lower() for tok in listOfTokens if len(tok) > 2] 



# def spamTest():
#     docList=[]; classList = []; fullText =[]
#     for i in range(1,26):
#         wordList = textParse(open('email/spam/%d.txt' % i,'r').read())
#         docList.append(wordList)
#         fullText.extend(wordList)
#         classList.append(1)
#         wordList = textParse(open('email/ham/%d.txt' % i,'r').read())
#         docList.append(wordList)
#         fullText.extend(wordList)
#         classList.append(0)
#     vocabList = createVocabList(docList) #创建词表
#     trainingSet = list(range(50)); testSet=[]           #create test set
#     for i in range(10):   # 筛选测试集
#         randIndex = int(random.uniform(0,len(trainingSet)))
#         testSet.append(trainingSet[randIndex])
#         del(trainingSet[randIndex])  
#     trainMat=[]; trainClasses = []
#     for docIndex in trainingSet:#train the classifier (get probs) trainNB0
#         trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
#         trainClasses.append(classList[docIndex])
#     p0V,p1V,pSpam = trainNB0(array(trainMat),array(trainClasses))
#     errorCount = 0
#     for docIndex in testSet:        #对测试集进行分类
#         wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
#         if classifyNB(array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:
#             errorCount += 1
#             print ("classification error",docList[docIndex])
#     print ('the error rate is: ',float(errorCount)/len(testSet))
#     #return vocabList,fullText



# ##################使用bayes从个人广告中获取区域倾向#############


# ###统计词汇表在文本中出现的次数，根据次数从高到低排序，最后筛选出最高的30个词######
# def calcMostFreq(vocabList,fullText):
#     import operator
#     freqDict = {}
#     for token in vocabList:
#         freqDict[token]=fullText.count(token)
#     sortedFreq = sorted(freqDict.items(), key=operator.itemgetter(1), reverse=True) 
#     return sortedFreq[:5]       

# def localWords(feed1,feed0):
#     import feedparser
#     docList=[]; classList = []; fullText =[]
#     minLen = min(len(feed1['entries']),len(feed0['entries']))
#     for i in range(minLen):
#         wordList = textParse(feed1['entries'][i]['summary'])
#         docList.append(wordList)
#         fullText.extend(wordList)
#         classList.append(1) #NY is class 1
#         wordList = textParse(feed0['entries'][i]['summary'])
#         docList.append(wordList)
#         fullText.extend(wordList)
#         classList.append(0)
#     vocabList = createVocabList(docList)#create vocabulary
#     top30Words = calcMostFreq(vocabList,fullText)   #remove top 30 words
#     for pairW in top30Words:
#         if pairW[0] in vocabList: vocabList.remove(pairW[0])
#     trainingSet = list(range(2*minLen)); testSet=[]           #create test set
#     for i in range(5):
#         randIndex = int(random.uniform(0,len(trainingSet)))
#         testSet.append(trainingSet[randIndex])
#         del(trainingSet[randIndex])  
#     trainMat=[]; trainClasses = []
#     for docIndex in trainingSet:#train the classifier (get probs) trainNB0
#         trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
#         trainClasses.append(classList[docIndex])
#     p0V,p1V,pSpam = trainNB0(array(trainMat),array(trainClasses))
#     errorCount = 0
#     for docIndex in testSet:        #classify the remaining items
#         wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
#         if classifyNB(array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:
#             errorCount += 1
#     print('the error rate is: ',float(errorCount)/len(testSet))
#     return vocabList,p0V,p1V


# def getTopWords(ny,sf):
#     import operator
#     vocabList,p0V,p1V=localWords(ny,sf)
#     topNY=[]; topSF=[]
#     for i in range(len(p0V)):
#         if p0V[i] > -6.0 : topSF.append((vocabList[i],p0V[i]))
#         if p1V[i] > -6.0 : topNY.append((vocabList[i],p1V[i]))
#     sortedSF = sorted(topSF, key=lambda pair: pair[1], reverse=True)
#     print ("SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**")
#     for item in sortedSF:
#         print (item[0])
#     sortedNY = sorted(topNY, key=lambda pair: pair[1], reverse=True)
#     print ("NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**")
#     for item in sortedNY:
#         print (item[0])



# import feedparser
# ny=feedparser.parse("http://www.nasa.gov/rss/dyn/image_of_the_day.rss")

# sf=feedparser.parse("http://sports.yahoo.com/nba/teams/hou/rss.xml")


# # print(len(ny['entries']))
# # print(ny['entries'][1])

# print(getTopWords(ny,sf))