机器学习实战 adaBoost

from numpy import *
from blaze import inf
from scipy.cluster.hierarchy import weighted


def loadSimpData():
    dataMat=matrix([[1.,2.1],[2.,1.1],[1.3,1.],[1.,1.],[2.,1.]])
    classLabels=[1.0,1.0,-1.0,-1.0,1.0]
    return dataMat,classLabels

def stumpClassify(dataMatrix,dimen,threshVal,threshIneq):          #一个阈值划分某一类标签（dim）的两边，小于阈值为-1，大于阈值为1.
    retArray=ones((shape(dataMatrix)[0],1))
    if threshIneq=='lt':
        retArray[dataMatrix[:,dimen]<=threshVal]=-1.0
    else:
        retArray[dataMatrix[:,dimen]>threshVal]=-1.0
    return retArray

def buildStump(dataArr,classLabels,D):
    dataMatrix=mat(dataArr)
    labelMat=mat(classLabels).T
    m,n=shape(dataMatrix)
    numSteps=10.0
    bestStump={}
    bestClasEst=mat(zeros((m,1)))
    minError=inf
    for i in range(n):
        rangeMin=dataMatrix[:,i].min()
        rangeMax=dataMatrix[:,i].max()
        stepSize=(rangeMax-rangeMin)/numSteps    #求步长
        for j in range(-1,int(numSteps)+1):       #就像书上的图那样，这部其实就是有一跟与X轴，Y轴垂直的线，按步长移动，分割两边的数据
            for inequal in ['lt','gt']:           #选那根分类结果最好的，保存到bestStump中
                threshVal=(rangeMin+float(j)*stepSize)  #阈值
                predictedVals=stumpClassify(dataMatrix, i, threshVal, inequal)
                errArr=mat(ones((m,1)))
                errArr[predictedVals==labelMat]=0
                weightedError=D.T*errArr
                print ("split: dim %d, thresh %.2f, thresh ineqal: %s, the weighted error is %.3f" % (i, threshVal, inequal, weightedError))
                if weightedError<minError:
                    minError=weightedError
                    bestClasEst=predictedVals.copy()
                    bestStump['dim']=i        #最合适类别
                    bestStump['thresh']=threshVal  #最合适阈值
                    bestStump['ineq']=inequal  #合适的lt,gt
    return bestStump,minError,bestClasEst
datMat,classLabels=loadSimpData()

'''
D=mat(ones((5,1))/5)
buildStump(datMat, classLabels, D)
'''
def adaBoostTrainDS(dataArr,classLabels,numIt=40):
    weakClassArr=[]
    m=shape(dataArr)[0]
    D=mat(ones((m,1))/m)
    aggClassEst=mat(zeros((m,1)))
    for i in range(numIt):
        bestStump,error,classEst=buildStump(dataArr,classLabels,D)
        print("D:",D.T)
        alpha=float(0.5*log((1.0-error)/max(error,1e-16)))  #确保不会除零导致溢出
        bestStump['alpha']=alpha                              
        weakClassArr.append(bestStump)
        print("classEst:",classEst.T)
        expon=multiply(-1*alpha*mat(classLabels).T,classEst)    #与类别标签相乘直接得到正负
        D=multiply(D,exp(expon))         #书中公式
        D=D/D.sum()
        aggClassEst+=alpha*classEst       #记录每个点的类别估计累计值
        print("aggClassEst: ",aggClassEst.T)
        print('------------------')
        #sign()!=mat那里得到的是true,false的矩阵，与ones((m,1))相乘后就可到0,1的矩阵
        aggErrors=multiply(sign(aggClassEst)!=mat(classLabels).T,ones((m,1))) #sign符号函数，大于0为1，小于0为-1
        errorRate=aggErrors.sum()/m
        print("total error: ",errorRate,"\n")
        if errorRate==0.0:
            break
    return weakClassArr,aggClassEst

'''
#测试完整adaBoostTrainDS
weakClassArr=adaBoostTrainDS(datMat, classLabels, 9)    
print(weakClassArr)
'''
def adaClassify(datToClass,classifierArr):
    dataMatrix=mat(datToClass)
    m=shape(dataMatrix)[0]
    aggClassEst=mat(zeros((m,1)))
    for i in range(len(classifierArr)):
        classEst=stumpClassify(dataMatrix, classifierArr[i]['dim'], classifierArr[i]['thresh'], classifierArr[i]['ineq'])
        aggClassEst+=classifierArr[i]['alpha']*classEst
        print(aggClassEst)
    return sign(aggClassEst)
'''
print('测试基于adaboost的分类')
weakClassArr=adaBoostTrainDS(datMat, classLabels, 9) 
adaClassify([0,0], weakClassArr)
'''

def loadDataSet(filename):
    numFeat=len(open(filename).readline().split('\t'))
    dataMat=[]
    labelMat=[]
    fr=open(filename)
    for line in fr.readlines():
        lineArr=[]
        curLine=line.strip().split('\t')
        for i in range(numFeat-1):
            lineArr.append(float(curLine[i]))
        dataMat.append(lineArr)
        labelMat.append(float(curLine[-1]))
    return dataMat,labelMat

'''
#adaboost测试马病数据集
datArr,labelArr=loadDataSet('C:/Users/xuwei/Desktop/机器学习/机器学习实战(pdf版+源码)/machinelearninginaction/Ch07/horseColicTraining2.txt')
classifierArray=adaBoostTrainDS(datArr, labelArr, 10)
testArr,testLabelArr=loadDataSet('C:/Users/xuwei/Desktop/机器学习/机器学习实战(pdf版+源码)/machinelearninginaction/Ch07/horseColicTest2.txt')
prediction10=adaClassify(testArr, classifierArray)
errArr=mat(ones((67,1)))
err=errArr[prediction10!=mat(testLabelArr).T].sum()
print(err)
'''

def plotROC(predStrengths,classLabels):
    import matplotlib.pyplot as plt
    cur=(1.0,1.0)
    ySum=0.0
    numPosClas=sum(array(classLabels)==1.0)
    yStep=1/float(numPosClas)   #numPosClas是正例数目，yStep是步长
    xStep=1/float(len(classLabels)-numPosClas)  #反例数目及步长
    sortedIndicies=predStrengths.argsort()
    fig=plt.figure()
    fig.clf()
    ax=plt.subplot(111)
    for index in sortedIndicies.tolist()[0]:
        if classLabels[index]==1.0:
            delX=0
            delY=yStep
        else:
            delX=xStep
            delY=0
            ySum+=cur[1]
        ax.plot([cur[0],cur[0]-delX],[cur[1],cur[1]-delY],c='b')
        cur=(cur[0]-delX,cur[1]-delY)
    ax.plot([0,1],[0,1],'b--')
    plt.xlabel('false posisive rate')
    plt.ylabel('true positive rate')
    plt.title('roc curve for adaBoost horse colic detection system')
    ax.axis([0,1,0,1])
    plt.show()
    print('the Area Under the Curve is:',ySum*xStep)
'''
ROC测试
datArr,labelArr=loadDataSet('C:/Users/xuwei/Desktop/机器学习/机器学习实战(pdf版+源码)/machinelearninginaction/Ch07/horseColicTraining2.txt')
classifierArray,aggClassEst=adaBoostTrainDS(datArr, labelArr, 10)
print('aggclassest',aggClassEst)
plotROC(aggClassEst.T,labelArr)
'''