【记录】AdaBoost基本逻辑的实现

*AdaBoost 5.6

基本逻辑的实现...

1.创建函数

# 数据集创建函数
def loadSimpleData():
    # 建议数据库建立
    datMat = np.matrix([[1.,2.1],
                    [2.,1.1],
                    [1.3,1.],
                    [1.,1.],
                    [2.,1.]])
    classLabels = [1.0,1.0,-1.0,-1.0,1.0]
    return datMat,classLabels

2.单层决策树生成函数

# 利用阈值比较来区分数据
def stumpClassify(dataMatrix,dimen,threshVal,threshIneq):
    # 阈值retArray初始化
    retArray = np.ones((np.shape(dataMatrix)[0],1))
    if threshIneq == 'lt':
        # 不大于阈值的情况
        retArray[dataMatrix[:,dimen] <= threshVal] = -1.0
    else:
        # 大于阈值的情况
        retArray[dataMatrix[:,dimen] > threshVal] = -1.0
    return retArray
    
# 遍历以找到数据集上最佳的单层决策树
def buildStump(dataArr,classLabels,D):
    dataMatrix = np.mat(dataArr)
    labelMat = np.mat(classLabels).T
    m,n = np.shape(dataMatrix)
    # 遍历特征所有可能值
    numSteps = 10.0
    # 存储给定权重向量D时，所得到的最佳单层决策树的相关信息
    bestStump = {}
    bestClasEst = np.mat(np.zeros((m,1)))
    # 寻找可能的最小错误率
    minError = np.inf 
    # 遍历所有值找到最佳的单层决策树
    for i in range(n):
        # rangeMin找到列中最小的值
        rangeMin = dataMatrix[:,i].min()
        # rangeMax找到列中最大的值
        rangeMax = dataMatrix[:,i].max()
        # 计算步长即每次循环的增量
        stepSize = (rangeMax-rangeMin)/numSteps
        for j in range(-1,int(numSteps)+1):
            # 在大于和小于之间切换不等式
            for inequal in ['lt', 'gt']: 
                threshVal = (rangeMin + float(j) * stepSize)
                # 调用stumpClassify对当前阈值进行分类
                predictedVals = stumpClassify(dataMatrix,i,threshVal,inequal)
                errArr = np.mat(np.ones((m,1)))
                # 计算当前阈值取值的误差
                errArr[predictedVals == labelMat] = 0
                weightedError = D.T*errArr  # 计算总误差乘以 D
                # 选出最小的误差，实质是选出最优的决策树的分类位置
                if weightedError < minError:
                    minError = weightedError
                    bestClasEst = predictedVals.copy()
                    bestStump['dim'] = i
                    bestStump['thresh'] = threshVal
                    bestStump['ineq'] = inequal
    return bestStump,minError,bestClasEst

3.输出分类结果和错误率

import numpy as np
D = np.mat(np.ones((5,1))/5)
datMat,classLabels = loadSimpleData()
buildStump(datMat,classLabels,D)

({'dim': 0, 'thresh': 1.3, 'ineq': 'lt'},
 matrix([[0.2]]),
 array([[-1.],
        [ 1.],
        [-1.],
        [-1.],
        [ 1.]]))

4.基于单层决策树的AdaBoost

# 输入数据集、类别标签、迭代次数
def adaBoostTrainDS(dataArr,classLabels,numIt):
    weakClassArr = []
    # 数据集中的数据点的数目
    m = np.shape(dataArr)[0]
    # 包含了每个数据点的权重
    D = np.mat(np.ones((m,1))/m)   
    # 记录每个数据点的类别估计累计值
    aggClassEst = np.mat(np.zeros((m,1)))
    # 迭代 numIt 次
    for i in range(numIt):
        bestStump,error,classEst = buildStump(dataArr,classLabels,D)
        # 学习器权重
        alpha = float(0.5*np.log((1.0-error)/max(error,1e-16)))
        # 当前权重记录
        bestStump['alpha'] = alpha  
        weakClassArr.append(bestStump)
        # 计算权重*真实值*预测值
        expon = np.multiply(-1*alpha*np.mat(classLabels).T,classEst) 
        # 更新每个预测值的权重
        D = np.multiply(D,np.exp(expon)) 
        # 归一化
        D = D/D.sum()
        # 所有分类器的 alc 训练误差
        aggClassEst += alpha*classEst
        # 最终分类器
        aggErrors = np.multiply(np.sign(aggClassEst) != np.mat(classLabels).T,np.ones((m,1)))
        # 错误率累计计算
        errorRate = aggErrors.sum()/m
        print("total error: ",errorRate)
        if errorRate == 0.0: break
    return weakClassArr,aggClassEst

5.分类结果

adaBoostTrainDS(datMat,classLabels,9)

total error:  0.2
total error:  0.2
total error:  0.0





([{'dim': 0, 'thresh': 1.3, 'ineq': 'lt', 'alpha': 0.6931471805599453},
  {'dim': 1, 'thresh': 1.0, 'ineq': 'lt', 'alpha': 0.9729550745276565},
  {'dim': 0, 'thresh': 0.9, 'ineq': 'lt', 'alpha': 0.8958797346140273}],
 matrix([[ 1.17568763],
         [ 2.56198199],
         [-0.77022252],
         [-0.77022252],
         [ 0.61607184]]))

6.基于AdaBoost的分类

# 利用训练出的多个弱分类器进行分类的函数
def adaClassify(datToClass,classifierArr):
    # 转换成NumPy矩阵
    dataMatrix = np.mat(datToClass)
    # 待分类样例个数
    m = np.shape(dataMatrix)[0]
    aggClassEst = np.mat(np.zeros((m,1)))
    for i in range(len(classifierArr)):
        #调用决策树
        classEst = stumpClassify(dataMatrix,classifierArr[i]['dim'],\
                                 classifierArr[i]['thresh'],\
                                 classifierArr[i]['ineq'])
        aggClassEst += classifierArr[i]['alpha']*classEst
        print(aggClassEst)
    return np.sign(aggClassEst)

adaClassify([0,0],classfierArr)