提升方法之AdaBoost算法

利用AdaBoost元算法提高分类性能

生成简单数据集

def loadSimpleData():
    dataMat = matrix([[1.,2.1],
                     [2.,1.1],
                     [1.3,1.],
                     [1.,1.],
                     [2.,1.],
                     ])
    classLabels = [1.0,1.0,-1.0,-1.0,1.0]
    return dataMat,classLabels

from numpy import *
dataMat,classLabels = loadSimpleData()

print(dataMat)
print(classLabels)

[[1.  2.1]
 [2.  1.1]
 [1.3 1. ]
 [1.  1. ]
 [2.  1. ]]
[1.0, 1.0, -1.0, -1.0, 1.0]

单层决策树生成函数

def stumpClassify(dataMatrix,dimen,threshVal,threshIneq):
    '''
    基于加权输入值
    通过阈值比较对数据进行分类
    并找到分类效果最好的单层决策树（只用一个特征的）'''
    retArray = ones((shape(dataMatrix)[0],1))
    if threshIneq =='lt':
        retArray[dataMatrix[:,dimen] <= threshVal] = -1.0
    else:
        retArray[dataMatrix[:,dimen] > threshVal] = -1.0
    return retArray

def buildStump(dataArr,classLabels,D):
    dataMatrix = mat(dataArr);labelMat = mat(classLabels).T
    m,n = shape(dataMatrix)
    numSteps = 10.0;bestStump = {};bestClassEst = mat(zeros((m,1)))
    minError = inf
    #对每个特征，最后取分类效果最好的那个特征
    for i in range(n):
        rangeMin = dataMatrix[:,i].min();rangeMax = dataMatrix[:,i].max()
        stepSize = (rangeMax - rangeMin)/numSteps
        #对每个步长
        for j in range(-1,int(numSteps)+1):
            #对每个不等号
            for inequal in ['lt','gt']:
                threshVal = (rangeMin + float(j) * stepSize)
                predictedVals = stumpClassify(dataMatrix,i,threshVal,inequal)
                errArr = mat(ones((m,1)))
                errArr[predictedVals == labelMat] = 0
                weightedError = D.T * errArr    #计算加权错误率
                #将当前错误率和已有最小错误率比较
                if weightedError < minError:
                    minError = weightedError
                    bestClassEst = predictedVals.copy()
                    bestStump['dim'] = i
                    bestStump['thresh'] = threshVal
                    bestStump['ineq'] = inequal
    return bestStump,minError,bestClassEst

D = mat(ones((5,1))/5)
buildStump(dataMat,classLabels,D)

({'dim': 0, 'thresh': 1.3, 'ineq': 'lt'}, matrix([[0.2]]), array([[-1.],
        [ 1.],
        [-1.],
        [-1.],
        [ 1.]]))

基于单层决策树的AdaBoost训练

def adaVoostTrainDS(dataArr,classLabels,numIt=40):
    weakClassArr = [] #储存弱分类器
    m = shape(dataArr)[0]
    D = mat(ones((m,1))/m)   #训练集初始权值分布
    aggClassEst  = mat(zeros((m,1)))  #记录每个数据点的类别估计累计值
    for i in range(numIt):
        bestStump,error,classEst = buildStump(dataArr,classLabels,D)  #classEst:估计的类别向量
        print('D:{}'.format(D.T))
        #max(error,1e-16)  
        #就是代表科学技术法，即1乘以10的-16次方。注意这里的1不能省略，因为可能造成歧义
        #确保在没有错误时不会发生除零溢出
        alpha = float(0.5*log((1.0-error)/max(error,1e-16)))   #计算当前分类器的系数
        bestStump['alpha'] = alpha
        weakClassArr.append(bestStump)
        print('classEst:{}'.format(classEst.T))
        #为下一次迭代更新D
        expon = multiply(-1*alpha*mat(classLabels).T,classEst)
        D = multiply(D,exp(expon))
        D = D/D.sum()
        #错误率累加计算，起到检验当前所有弱分类器组合的强分类器的效果的作用
        aggClassEst += alpha*classEst
        print('aggClasssEst:{}'.format(aggClassEst.T))
        aggErrors = multiply(sign(aggClassEst) != mat(classLabels).T,ones((m,1)))
        errorRate = aggErrors.sum()/ m
        print('total error:{}'.format(errorRate))
        if errorRate == 0.0:
            break
    #在画ROC曲线时测试
    #return weakClassArr,aggClassEst
    return weakClassArr

classifierArray = adaVoostTrainDS(dataMat,classLabels,9)

D:[[0.2 0.2 0.2 0.2 0.2]]
classEst:[[-1.  1. -1. -1.  1.]]
aggClasssEst:[[-0.69314718  0.69314718 -0.69314718 -0.69314718  0.69314718]]
total error:0.2
D:[[0.5   0.125 0.125 0.125 0.125]]
classEst:[[ 1.  1. -1. -1. -1.]]
aggClasssEst:[[ 0.27980789  1.66610226 -1.66610226 -1.66610226 -0.27980789]]
total error:0.2
D:[[0.28571429 0.07142857 0.07142857 0.07142857 0.5       ]]
classEst:[[1. 1. 1. 1. 1.]]
aggClasssEst:[[ 1.17568763  2.56198199 -0.77022252 -0.77022252  0.61607184]]
total error:0.0

AdaBoost的分类函数

def adaClassify(dataToClass,classifierArr):
    dataMatrix = mat(dataToClass)
    m = shape(dataMatrix)[0]
    aggClassEst = mat(zeros((m,1)))
    for i in range(len(classifierArr)):
        classEst = stumpClassify(dataMatrix,classifierArr[i]['dim'],classifierArr[i]['thresh'],classifierArr[i]['ineq'])
        aggClassEst += classifierArr[i]['alpha']*classEst
        print(aggClassEst)
    return sign(aggClassEst)

adaClassify([[5,5],[0,0]],classifierArray)

[[ 0.69314718]
 [-0.69314718]]
[[ 1.66610226]
 [-1.66610226]]
[[ 2.56198199]
 [-2.56198199]]





matrix([[ 1.],
        [-1.]])

在马疝病预测中应用分类器

def loaddataSet(fileName):
    numFeat = len(open(fileName).readline().split('\t'))
    dataMat = [];labelMat = []
    fr = open(fileName)
    for line in fr.readlines():
        lineArr = []
        curLine = line.strip().split('\t')
        for i in range(numFeat-1):
            lineArr.append(float(curLine[i]))
        dataMat.append(lineArr)
        labelMat.append(float(curLine[-1]))
    return dataMat,labelMat

dataArr,labelArr = loaddataSet('E:\\DataMining\\资料\\machinelearninginaction\\Ch07\\horseColicTraining2.txt')
classifierArr = adaVoostTrainDS(dataArr,labelArr,10)

D:[[0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.00334448
  0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.00334448
  0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.00334448
  0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.00334448
   0.05809528 -0.65208904 -1.02662219  0.27123572  0.5606821 ]]
total error:0.23076923076923078

testArr,testLabelArr = loaddataSet('E:\\DataMining\\资料\\machinelearninginaction\\Ch07\\horseColicTest2.txt')
prediction10 = adaClassify(testArr,classifierArray)

[[ 0.69314718]
 [ 0.69314718]
 [-0.69314718]
 [-0.69314718]
 [ 0.69314718]
 [-0.69314718]
 [ 0.69314718]
 [ 0.69314718]
 [ 0.69314718]
 [-0.77022252]
 [ 0.61607184]
 [ 0.61607184]]

测试的错误率

errArr = mat(ones((67,1)))
errArr[prediction10 != mat(testLabelArr).T].sum()/67

0.417910447761194

非均衡分类问题

ROC曲线的绘制和AUC计算函数

def plotROC(predStrengths,classLabels):
    import matplotlib.pyplot as plt
    cur = (1.0,1.0)  #当前光标的位置
    ySum = 0.0    #计算AUC的值
    numPosClas = sum(array(classLabels) == 1.0)   #计算正例的数目
    yStep = 1/float(numPosClas)
    xStep = 1/float(len(classLabels) - numPosClas)
    sortedIndicies = predStrengths.argsort()  #获取排好序的索引
    fig = plt.figure()
    fig.clf()
    ax = plt.subplot(111)
    for index in sortedIndicies.tolist()[0]:
        if classLabels[index] ==1.0:
            delX = 0;delY= yStep
        else:
            delX = xStep;delY = yStep
            ySum += cur[1]
        ax.plot([cur[0],cur[0]-delX],[cur[1],cur[1]-delY],c='b')
        cur = (cur[0]-delX,cur[1]-delY)
    ax.plot([0,1],[0,1],'b--')
    plt.xlabel('False Positive Rate');plt.ylabel('True Positive Rate')
    plt.title('ROC curve for AdaBoost Horse Colic Detection System')
    ax.axis([0,1,0,1])
    plt.show()
    print('the Area Under the Curve is{}'.format(ySum*xStep))

测试

classifierArr,aggClassEst =  adaVoostTrainDS(dataArr,labelArr,10)

D:[[0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.00334448
  0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.00334448
  0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.00334448
  0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.00334448
  0.00334448 0.00334448 0.00334448 0.00334448 0.00334448 0.0033444
  -0.01293737  1.53203035  0.95841088 -1.04249592  0.23749438  0.5606821
   1.20719077  0.91726555 -0.10329743 -0.57967867  0.27123572  1.69342306
   0.05809528 -0.65208904 -1.02662219  0.27123572  0.5606821 ]]
total error:0.23076923076923078

plotROC(aggClassEst.T,labelArr)

the Area Under the Curve is0.5212183118209663

测试可以看出分类效果并不理想