统计学习方法——提升方法（三）

提升方法（三）

现在我们来看一个AdaBoost算法的简单实现。

数据

这里我们采用几个简单数据作为训练数据：

######训练样本######
def load_simple_data():
    data_mat = np.matrix([[1.0,2.1],[2.0,1.1],[1.3,1.0],[1.0,1.0],[2.0,1.0]])
    class_labels=[1.0,1.0,-1.0,-1.0,1.0]
    return data_mat,class_labels

实现

######根据阈值判断每个特征的输出值######
#ret_array初始化时不能全1或-1
def stump_classify(data_matrix,dimen,threshval,thresh_ineq):
    ret_array=zeros((shape(data_matrix)[0],1))
    if thresh_ineq=="lt":
        ret_array[data_matrix[:,dimen]<=threshval]=-1.0
    else:
        ret_array[data_matrix[:,dimen]>threshval]=1.0
    return ret_array   

 ######实现单层决策树######
#lt_predicted_arr存储小于等于阈值的输出值，gt_predicted_arr存储大于阈值的输出值
def build_stump(data_arr,class_labels,D):
    data_matrix=mat(data_arr)   #数据集
    label_mat=mat(class_labels).T  #标签集
    m,n=shape(data_matrix)   #数据集的行、列
    num_steps=10.0      #用于设置划分点
    best_stump={}
    best_class_est=mat(zeros((m,1)))
    min_error=inf
    
    for i in range(n):
        range_min=data_matrix[:,i].min()   #数据集中某一列的最小值
        range_max=data_matrix[:,i].max()   #数据集中某一列的最大值
        step_size=(range_max-range_min)*1.0/num_steps   #用于设置划分点
        for j in range(-1,int(num_steps)+1):
            thresh_val=(range_min+float(j)*step_size)   #计算其中一个划分点
            lt_predicted_arr=zeros((m,1))     #获得析域不等号的值
            gt_predicted_arr=zeros((m,1))     #获得大于不等号的值
            predicted_arr=zeros((m,1))        #获得最终的预测值
            for inequal in ["lt","gt"]:
                preducted_vals=stump_classify(data_matrix,i,thresh_val,inequal)
                if inequal=='lt':
                    lt_predicted_arr=preducted_vals
                else:
                    gt_predicted_arr=preducted_vals
            for k in range(m):
                predicted_arr[k] = lt_predicted_arr[k]
                if(gt_predicted_arr[k] != 0):
                    predicted_arr[k] = gt_predicted_arr[k]
            err_arr = mat(ones((m, 1)))
            err_arr[predicted_arr == label_mat] = 0
            weight_error = D.T * err_arr
            print("min_error = %0.5f, split: dim %d, thresh %0.2f,the weighted error is %0.3f" %(min_error, i, thresh_val, weight_error))
            if(weight_error < min_error):
                min_error = weight_error
                best_class_est = predicted_arr.copy()
                best_stump["dim"] = i
                best_stump["thresh"] = thresh_val
                best_stump["class_est"] = best_class_est
    return best_stump,min_error,best_class_est

###根据单层决策树得到一个弱分类器保存在列表中
###根据得到的弱分类器输出的预测值和原始值修改权重，减小误差小的点的权重，增大误差大的点的权重。最后误差率等于0则停止迭代。
def adaboost_train_DS(data_arr, class_labels, num_it = 40):
    weak_class_arr = []
    m = shape(data_arr)[0]
    D = mat(ones((m, 1)) / m)
    agg_class_est = mat(zeros((m, 1)))
    counts = 0

    for i in range(num_it):
        best_stump,error,class_est = build_stump(data_arr, class_labels, D)
        print("D: ", D.T)
        alpha = float(0.5 * log((1.0 - error) / error))
        best_stump["alpha"] = alpha
        weak_class_arr.append(best_stump)
        print("class_est: ", class_est.T)
        expon = multiply(-1 * alpha * mat(class_labels).T, class_est)
        D = multiply(D, exp(expon))
        D = D / D.sum()
        agg_class_est += alpha * class_est  #预测值
        print("agg_class_est: ", agg_class_est.T)
        agg_errors = multiply(sign(agg_class_est) != mat(class_labels).T, ones((m, 1)))
        error_rate = agg_errors.sum() / m
        print("total error: ", error_rate)
        counts += 1
        if(error_rate == 0.0):
            break
    return weak_class_arr,counts

###弱分类器：根据每一个弱分类器阈值，输出每个特征的输出值。
def Gx(data_matrix, reshval):
    m = shape(data_matrix)[0]
    result_data = mat(ones((m, 1)))
    result_data[data_matrix[:, 0] <= reshval] = -1.0
    result_data[data_matrix[:, 0] > reshval] = 1.0
    return result_data

###根据输出数据，输出实际预测值。
def ada_classfiy(input_data, weak_class_arr, counts):
    data_matrix = mat(input_data)
    m,n = shape(data_matrix)  
    agg_class_est = mat(zeros((m, 1)))
    for j in range(n):
        for i in range(len(weak_class_arr)):
            agg_class_est += (weak_class_arr[i]["alpha"] * (Gx(input_data[:, j], weak_class_arr[i]["thresh"])).T).T
    print(agg_class_est)
    return sign(agg_class_est)

###画图###
def experiment_plot(data_matrix, agg_class_est):
    data_arr_in = data_matrix.getA()
    label_arr_in = agg_class_est.getA()
    m,n = shape(data_matrix)

    for i in range(m):
        if(label_arr_in[i, 0] == -1):
            plt.plot(data_arr_in[i, 0], data_arr_in[i, 1], "ob")
        elif(label_arr_in[i, 0] == 1):
            plt.plot(data_arr_in[i, 0], data_arr_in[i, 1], "or")

    plt.xlabel("X")
    plt.ylabel("Y")
    plt.show()

测试与结果

def main():
    #D = mat(ones((5, 1)) / 5.0)
    data_mat,class_labels = load_simple_data()
    #build_stump(data_mat, class_labels, D)
    weak_class_arr,counts = adaboost_train_DS(data_mat, class_labels)
    print("**************************")
    data_mat2 = matrix([[1.2, 2.0],
                       [1.0, 1.0],
                       [0.6, 1.2],
                       [0.8, 1.1],
                       [1.8, 1.0],
                       [2.0, 0.4],
                       [1.7, 0.8],
                       [3.5, 0.2],
                       [2.5, 0.8],
                       [2.8, 0.9]])
    agg_class_est = ada_classfiy(data_mat2, weak_class_arr, counts)
    print("--------------------------")
    print(agg_class_est)
    experiment_plot(data_mat2, agg_class_est)

main()

结果如下所示：

---

min_error = inf, split: dim 0, thresh 0.90,the weighted error is 0.400
min_error = 0.40000, split: dim 0, thresh 1.00,the weighted error is 0.400
min_error = 0.40000, split: dim 0, thresh 1.10,the weighted error is 0.400
min_error = 0.40000, split: dim 0, thresh 1.20,the weighted error is 0.400
min_error = 0.40000, split: dim 0, thresh 1.30,the weighted error is 0.200
min_error = 0.20000, split: dim 0, thresh 1.40,the weighted error is 0.200
min_error = 0.20000, split: dim 0, thresh 1.50,the weighted error is 0.200
min_error = 0.20000, split: dim 0, thresh 1.60,the weighted error is 0.200
min_error = 0.20000, split: dim 0, thresh 1.70,the weighted error is 0.200
min_error = 0.20000, split: dim 0, thresh 1.80,the weighted error is 0.200
min_error = 0.20000, split: dim 0, thresh 1.90,the weighted error is 0.200
min_error = 0.20000, split: dim 0, thresh 2.00,the weighted error is 0.600
min_error = 0.20000, split: dim 1, thresh 0.89,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.00,the weighted error is 0.200
min_error = 0.20000, split: dim 1, thresh 1.11,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.22,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.33,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.44,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.55,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.66,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.77,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.88,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 1.99,the weighted error is 0.400
min_error = 0.20000, split: dim 1, thresh 2.10,the weighted error is 0.600
D: [[0.2 0.2 0.2 0.2 0.2]]
class_est: [[-1. 1. -1. -1. 1.]]
agg_class_est: [[-0.69314718 0.69314718 -0.69314718 -0.69314718 0.69314718]]
total error: 0.2
min_error = inf, split: dim 0, thresh 0.90,the weighted error is 0.250
min_error = 0.25000, split: dim 0, thresh 1.00,the weighted error is 0.625
min_error = 0.25000, split: dim 0, thresh 1.10,the weighted error is 0.625
min_error = 0.25000, split: dim 0, thresh 1.20,the weighted error is 0.625
min_error = 0.25000, split: dim 0, thresh 1.30,the weighted error is 0.500
min_error = 0.25000, split: dim 0, thresh 1.40,the weighted error is 0.500
min_error = 0.25000, split: dim 0, thresh 1.50,the weighted error is 0.500
min_error = 0.25000, split: dim 0, thresh 1.60,the weighted error is 0.500
min_error = 0.25000, split: dim 0, thresh 1.70,the weighted error is 0.500
min_error = 0.25000, split: dim 0, thresh 1.80,the weighted error is 0.500
min_error = 0.25000, split: dim 0, thresh 1.90,the weighted error is 0.500
min_error = 0.25000, split: dim 0, thresh 2.00,the weighted error is 0.750
min_error = 0.25000, split: dim 1, thresh 0.89,the weighted error is 0.250
min_error = 0.25000, split: dim 1, thresh 1.00,the weighted error is 0.125
min_error = 0.12500, split: dim 1, thresh 1.11,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 1.22,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 1.33,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 1.44,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 1.55,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 1.66,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 1.77,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 1.88,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 1.99,the weighted error is 0.250
min_error = 0.12500, split: dim 1, thresh 2.10,the weighted error is 0.750
D: [[0.5 0.125 0.125 0.125 0.125]]
class_est: [[ 1. 1. -1. -1. -1.]]
agg_class_est: [[ 0.27980789 1.66610226 -1.66610226 -1.66610226 -0.27980789]]
total error: 0.2
min_error = inf, split: dim 0, thresh 0.90,the weighted error is 0.143
min_error = 0.14286, split: dim 0, thresh 1.00,the weighted error is 0.357
min_error = 0.14286, split: dim 0, thresh 1.10,the weighted error is 0.357
min_error = 0.14286, split: dim 0, thresh 1.20,the weighted error is 0.357
min_error = 0.14286, split: dim 0, thresh 1.30,the weighted error is 0.286
min_error = 0.14286, split: dim 0, thresh 1.40,the weighted error is 0.286
min_error = 0.14286, split: dim 0, thresh 1.50,the weighted error is 0.286
min_error = 0.14286, split: dim 0, thresh 1.60,the weighted error is 0.286
min_error = 0.14286, split: dim 0, thresh 1.70,the weighted error is 0.286
min_error = 0.14286, split: dim 0, thresh 1.80,the weighted error is 0.286
min_error = 0.14286, split: dim 0, thresh 1.90,the weighted error is 0.286
min_error = 0.14286, split: dim 0, thresh 2.00,the weighted error is 0.857
min_error = 0.14286, split: dim 1, thresh 0.89,the weighted error is 0.143
min_error = 0.14286, split: dim 1, thresh 1.00,the weighted error is 0.500
min_error = 0.14286, split: dim 1, thresh 1.11,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 1.22,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 1.33,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 1.44,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 1.55,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 1.66,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 1.77,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 1.88,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 1.99,the weighted error is 0.571
min_error = 0.14286, split: dim 1, thresh 2.10,the weighted error is 0.857
D: [[0.28571429 0.07142857 0.07142857 0.07142857 0.5 ]]
class_est: [[1. 1. 1. 1. 1.]]
agg_class_est: [[ 1.17568763 2.56198199 -0.77022252 -0.77022252 0.61607184]]
total error: 0.0

[[ 3.73766962e+00]
[-1.54044504e+00]
[-1.38629436e+00]
[-1.38629436e+00]
[ 1.79175947e+00]
[-2.22044605e-16]
[-2.22044605e-16]
[-2.22044605e-16]
[-2.22044605e-16]
[-2.22044605e-16]]

[[ 1.]
[-1.]
[-1.]
[-1.]
[ 1.]
[-1.]
[-1.]
[-1.]
[-1.]
[-1.]]

---

完整代码

完整代码：https://github.com/canshang/-/blob/master/AdaBoost.ipynb

参考文献

《统计学习方法》
《机器学习实战》
https://www.jianshu.com/p/2ea2f4ec121c?t=123&utm_source=oschina-app