AdaBoost算法的单机实现

AI课上的作业,我用python实现一下

实现的目标如下:

首先简单介绍一下Adaboost算法,也叫做自增强适应。用来将多个弱的分类器/预测器变成一个比较强的分类器。具体算法如下:

[0]弱分类器我只实现了简单的二分类,然后用AdaBoost算法将这些分类器训练成一个强分类器。

[1]强分类器由多个弱分类器加权之后结合而成,每一个弱分类器则有自己的判别规则。这里简单起见只用了最最简单的水平|垂直分类器。

[2]弱分类器是一个闭包函数,绑定了相关参数,没有使用类。

[3]强分类器由多个弱分类器组成,也是一个闭包函数

[4]中间很多语句是用来输出参数的,所以看着可能稍微有一点乱。

[5]关于弱分类器的选择,将数据点离散化之后就构造出许多弱分类器,利用误差公式来选择一个最好的。

[6]将数据分成了train和test…因为数据实在太少了,我也懒得搞了。。。

import matplotlib.pyplot as plt
import math

blue = [(-2,19),(13,15),(-8,11),(-14,14),(6,19),(-12,4),(15,13)]#,(26,13),(24,9)]
red  = [(4,-3),(5,-2),(1,6),(-6,5),(5,3),(11,-5),(-7,9)]#,(6,-3)]#,(15,-1),(16,-8)]
test1 = [((26,13),-1),((24,9),-1),((15,-1),1),((16,-8),1),((6,-3),1)]
points = [(point,1)for point in red]
points.extend((point,-1) for point in blue)
e = math.e

def create_weak_classifiers(points):
    def create_weak_classifier(_p,_theta,_kind):
        #val is input,theta is threshold,p determines area
        def weak_classifier(point):
            val = point[0] if _kind == 1 else point[1]
            if _p*val<_p*_theta:
                return 1
            else:
                return -1
        return weak_classifier

    #remove repeated position and sort
    x_index = list(set(sorted(p[0][0]  for p in points)))
    y_index = list(set(sorted(p[0][1]  for p in points)))
    n = len(x_index)
    h = {}
    attributes = {}
    ind = 0

    for pos,t in zip((x_index,y_index),(1,2)):
        for i in range(len(pos)-1):
            for p in (-1,1):
                m = (pos[i]+pos[i+1])/2
                h[ind] = create_weak_classifier(p,m,t)
                attributes[ind] = (p,m,t)
                ind +=1
    return h,attributes
def create_strong_classifier(H):
    def strong_calssifier(point):
        ret = 0
        for alpha,ind,weak_cls in H:
            ret += alpha*weak_cls(point)
        return 1 if ret>0 else -1#sign(ret)
    return strong_calssifier
def compute_error(points,f,D):
    error = 0
    for (i,(point,kind)) in enumerate(points):
        error += D[i]*(kind!=f(point))

    return error
def select_best_weak(points,h,D):
    min_error = 10**3
    ind,f = None,None
    for index,weak_cls in h.items():
        error = compute_error(points,weak_cls,D)
        if error < min_error:
            min_error = error
            ind,best_cls = index,weak_cls
    return ind,best_cls,min_error

#initialize parameter
N = len(blue)+len(red)
D = [1/N]*N 
T = 10
alpha = [0]*T
h,attributes = create_weak_classifiers(points)
H = []
if __name__ == '__main__':
    for t in range(T):
        print('Iteration{} :'.format(t),end='  ')
        ind,current_weak,error = select_best_weak(points,h,D)
        assert(error<0.5)
        alpha[t] = 0.5*math.log((1-error)/error)
        H.append((alpha[t],ind,current_weak))
        print('alpha :{}'.format(alpha[t]),end='\t')
        #update D
        Z = sum(D[i]*e**(-alpha[t]*points[i][1]*current_weak(points[i][0])) for i in range(N))
        for i in range(N):
            point,kind = points[i]
            frac_1 = D[i]*e**(-alpha[t]*kind*current_weak(point))
            D[i] = frac_1/Z

        #print h(x)
        attr = attributes[ind]
        p,m,kind = attr[0],attr[1],attr[2]
        print('h(x):{} * {} < {}'.format(p,'x' if kind==1 else 'y',m),end='\t')
        #compute the CE
        strong_cls = create_strong_classifier(H)
        total_error = compute_error(points,strong_cls,D)
        print('CE:{0:.3f}'.format(total_error))
        if total_error==0:
            count = 0
            for point,kind in test1:
                count += (kind!=strong_cls(point))
            print('test data'count)
            break