机器学习07-GBDT-python

github简单实现

一. 原理推导

二. python实现

1. 回归

1. 计算f初始值

# 步骤1：初始化f(x) 的初始值
    def initFValue(self , targets):
    # 这里简单直接初始化为0，当然对平方损失函数，这里可以使用平均值来
        self.fValue = [0 for i in targets]
        #avg = sum(targets) / len(targets)
        #self.fValue = [avg for i in targets]

2. 计算负梯度

    def calaResidual(self, yReal , fValue):
#        ret = [0 for f in yReal]
#        for i in range(len(yReal)):
#            ret[i] = yReal[i] - yPre[i]
        
        ret = [ i-y for i,y in zip(yReal,fValue) ]

3. 更新叶子节点值

    def updateLeafValue(self ,tree, dataSet,target):
        pass
        # 因为我是借用cart回归树的叶子值，而回归树叶子节点的值就是平均值

4. 更新f值

    def updateFValue(self,dataSet , tree):
        # 迭代一轮就把叶子节点的值相加
        for i in range(len(self.fValue)):
            data = dataSet[i]
            self.fValue[i] += cart.cartReg().predict0(data , tree)

5. 训练模型

    def fit(self , dataSet , target):
        # 1 初始化f0
        self.initFValue(target)
        
        for i  in range(self.nTrees):
            # 2 计算负梯度
            newTarget = self.calaResidual(target , self.fValue)
            print newTarget[0]
            # 3 构造回归树
            tree = cart.cartReg().buildTree(dataSet ,newTarget, self.depth , self.leafSize)
            # 4 拟合叶子节点
            self.updateLeafValue(tree , dataSet , newTarget)
            # 5 更新f值
            self.updateFValue(dataSet  , tree)
            
            self.allTrees.append(tree)
            self.lost.append(self.calaLost(target))
        return self

6. 预测

    def predict(self , predList):
        value = 0.0
        for tree in self.allTrees:
        	# 每棵树对应叶子节点值相加就是需要预测的值
            value +=  cart.cartReg().predict0(predList, tree)
            #print value
        return value

2. 二分类gbdt

1.标签为 {0,1}的分类问题

1. 初始化f0值

    def init_f_value(self, target):
        '''
        初始化f0的值
        :param target:
        :return:
        '''
        z = sum(target) / sum([1 - y for y in target])
        self.fValue = [math.log(z) for i in target]
        
        return self

2. 计算负梯度

@staticmethod
def sigmod(z):
    return 1.0 / (1.0 + (float)(math.exp(-z)))

def computer_residual(self, target):
    return [y - self.sigmod(z) for y, z in zip(target, self.fValue)]

3. 拟合叶子节点

    def calculate_leaf_value(self, target, index):
        """
        1. 计算叶子节点值
        2. 更新f值
        :param target:
        :param index: 叶子节点索引
        :return: 叶子节点值
        """
        # 更新叶子节点值
        y = [target[i] for i in index]
        fn = [self.fValue[i] for i in index]
        p = [self.sigmod(f) for f in fn]
        residual = sum(yi - pi for yi, pi in zip(y, p))
        prob = sum([pi * (1 - pi) for pi in p])
        # 更新f值
        value = residual / prob
        self.update_f_value(value, index)
        return value
   def update_leaf_value(self, tree, data_set, target):
        spilt_index = tree['spIndex']
        spilt_value = tree['spValue']
        index_left, index_right = self.spilt_data(data_set, spilt_index, spilt_value)
        data_left = [data_set[i] for i in index_left]
        data_right = [data_set[i] for i in index_right]
        if isinstance(tree['left'], dict):
            self.update_leaf_value(tree['left'], data_left, target)
        else:
            tree['left'] = self.calculate_leaf_value(target, index_left)
        
        if isinstance(tree['right'], dict):
            self.update_leaf_value(tree['right'], data_right, target)
        else:
            tree['right'] = self.calculate_leaf_value(target, index_right)

4. 更新f值

    def update_f_value(self, value, index):
        for i in index:
            self.fValue[i] += value

5. 训练模型

    def fit(self, data_set, target):
        self.init_f_value(target)
        for i in range(self.n_trees):
            new_target = self.computer_residual(target)
            tree = cart.cartReg().buildTree(data_set, new_target, self.depth, self.leaf_size)
            self.update_leaf_value(tree, data_set, target)
            self.all_trees.append(tree)
            lost = self.compute_lost(target)
            self.losts.append(lost)

6. 预测

    def predict_prob(self, predict_vector):
        fn = 0.0
        for tree in self.all_trees:
            fn += cart.cartReg().predict0(predict_vector, tree)
        return self.sigmod(fn)
    
    def predict(self, predict_vector, thr=0.5):
        prob = self.predict_prob(predict_vector)
        if prob > thr:
            return 1
        return 0

2.标签为{-1,1}的分类问题

1. 初始化f0值

    def initFValue(self, target):
        self.fValue = [t / (abs(t) * (2 - abs(t))) for t in target]

2. 计算负梯度

    def calaResidual(self, yReal, fValue):
        target = [2.0 * y / (1 + math.exp(2 * y * f)) for y, f in zip(yReal, fValue) ]
        return target

3. 拟合叶子节点

    def calaLeafValue(self , target):
        sum0 = 0.0
        for t in target:
            sum0 += abs(t) * (2 - abs(t))
            if sum0 == 0:
                pass
        return sum(target) / sum0
    
    def updateLeafValue(self, tree, dataSet, target):
        # 通过tree字典去切分数据
        
        spIndex = tree['spIndex']
        spValue = tree['spValue']
        subLeft, subRight, tarLeft, tarRight = cart.cartReg.spiltData(dataSet, target, spIndex, spValue)
        if not isinstance(tree['left'], dict):
            tree['left'] = self.calaLeafValue(tarLeft)
        else:
            self.updateLeafValue(tree['left'], subLeft, tarLeft)
        
        if not isinstance(tree['right'], dict):
            tree['right'] = self.calaLeafValue(tarRight)
        else:
            self.updateLeafValue(tree['right'], subRight, tarRight)

4. 更新f值

    def updateFValue(self, dataSet , tree):
        for i in range(len(self.fValue)):
            data = dataSet[i]
            self.fValue[i] += cart.cartReg().predict0(data , tree)

5. 训练模型

    def fit(self , dataSet , target):
        # 1 初始化f0
        self.initFValue(target)
        
        for i  in range(self.nTrees):
            # 2 计算负梯度
            newTarget = self.calaResidual(target , self.fValue)
            print newTarget[0]
            # 3 构造回归树
            tree = cart.cartReg().buildTree(dataSet ,newTarget, self.depth , self.leafSize)
            # 4 拟合叶子节点
            self.updateLeafValue(tree , dataSet , newTarget)
            # 5 更新f值
            self.updateFValue(dataSet  , tree)
            
            self.allTrees.append(tree)
            self.lost.append(self.calaLost(target))
        return self

6. 预测

    def predict(self, predList, threshold = 0.5):
        p = self.predict_prob(predList)
        if p >= threshold:
            return 1
        return -1
    
    def predict_prob(self, predList):
        value = super(GBDTClissity, self).predict(predList)
        # print value
        p = 1 / (1 + math.exp(-value))
        # print p
        return p

三. sklearn