GBDT手动实现，纯代码

本人刚开始开始机器学习，在关于gbdt的实现上，总是想着重现gbdt的官方接口，但由于没找到有用的资料硬生生纠结了一个星期
gbdt回归就是简单的用cart回归取迭代（负梯度）残差，今天这里要实现的是gbdt二分类，代码如下：

# -- encoding:utf-8 --
"""
Create by ibf on 2019/11/18
"""

import numpy as np
from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
from sklearn.metrics import r2_score, accuracy_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier,GradientBoostingRegressor
from sklearn import tree
import pydotplus

#适应于gbdt的cart树，输出为f的更新值，而不是概率的预测值
class Try_GBDT():
    def __init__(self,num_round=10,eta=0.1,gamma=0,Lambda=1,scoring="mse",y_true=[]):
        self.num_round = num_round
        self.eta = eta
        self.ensemble = []
        self.haty = None 
        self.f = None 
        self.y_true=y_true
    def _Gain(self,listL,listR,X_train,y_train):
        """计算Gain，方便确定最佳分枝"""
        #print(listL,listR,X_train)
        return np.var(y_train)-len(listL)/X_train.shape[0]*np.var(y_train[listL])-len(listR)/X_train.shape[0]*np.var(y_train[listR])
    def _w(self,indexlist,y_train):
            y_r=self.y_true[indexlist]-y_train[indexlist]
            return np.sum(y_train[indexlist])/np.sum(y_r.dot(1-y_r))
    def fit(self,X_train):
        
        def BestSplit(X_train,indexlist):
            """寻找最佳切分，如果有最佳切分，返回切分特征和切分值;如果无最佳切分，返回None"""
            bestGain = 0
            bestSplitFeature = -1
            bestSplitValue = -1
            #print('x',X_train)
            for Feature in range(X_train.shape[1]):
                #print('feature',Feature)
                #ValueSet = set(X_train[:,Feature])
                #ValueSet=np.sort(X_train[:,Feature].unique())
                ValueSet=sorted(set(X_train[:,Feature]))
                #print('valueset',ValueSet)
                for Val in ValueSet:
                    if Val==ValueSet[-1]:break
                    Val=(Val+ValueSet[int(np.argwhere(ValueSet==Val))+1])/2
                    boolindexLeft = X_train[:,Feature] <= Val
                    boolindexRight = ~boolindexLeft
                    #print('boolindex',boolindexLeft,boolindexRight,indexlist)
                    indexLeft = indexlist[boolindexLeft]
                    indexRight = indexlist[boolindexRight]
                    gain = self._Gain(indexLeft,indexRight,X_train,y_train)
                    if gain > bestGain and len(indexLeft)>1 and len(indexRight)>1:
                        bestGain = gain
                        bestSplitFeature = Feature
                        bestSplitValue = Val
            if bestSplitFeature == -1:
                return None,None
            else:
                return bestSplitFeature,bestSplitValue
        def create_tree(X_train,y_train,indexlist = np.arange(len(X_train)),depth=0):
            """建立新树，以字典形式保存，并更新self.f(这次更新后每个样本的目标分数)"""
            bestSplitFeature,bestSplitValue = BestSplit(X_train,indexlist)
            #print('bestSplitFeature',bestSplitFeature,bestSplitValue)
            if bestSplitFeature is None or depth>=1:
                w = self._w(indexlist,y_train)
                self.f[indexlist] = w
                return w
            else :
                depth+=1
                left_index = X_train[:,bestSplitFeature] <= bestSplitValue
                sub_X_train_left = X_train [left_index]
                sub_X_train_right = X_train [~left_index] 
                indexlist_left = indexlist[left_index]
                indexlist_right = indexlist[~left_index]
                #print('depth',depth)
                #print('indexlistleftandright',indexlist_left,indexlist_right)
                leftchild = create_tree(sub_X_train_left,y_train,indexlist_left,depth)
                rightchild = create_tree(sub_X_train_right,y_train,indexlist_right,depth)
                return {bestSplitFeature:{"<={}".format(bestSplitValue): leftchild,">{}".format(bestSplitValue): rightchild}}
        self.haty = np.zeros(len(X_train))+np.log(6.0/4)
        for _ in range(self.num_round):
            self.f = np.empty(len(X_train))
            y_train=self.y_true-1/(1+np.exp(-self.haty))
            newtree = create_tree(X_train,y_train,np.arange(len(X_train)),0)
            self.ensemble.append(newtree)
            #print('ensemble',self.ensemble)
            self.haty = self.haty + self.eta*self.f
        return self.haty,y_train,newtree
    def draw_one_tree(self,index):
        from graphviz import Digraph

        def export_graphviz(tree,root_index): 
            root = next(iter(tree))
            text_node.append([str(root_index),"feature:{}".format(root)])
            secondDic = tree[root]
            for key in secondDic:
                if type(secondDic[key]) == dict:
                    i[0] += 1
                    secondrootindex=i[0]
                    text_edge.append([str(root_index),str(secondrootindex),str(key)])
                    export_graphviz(secondDic[key],secondrootindex)
                else:
                    i[0] += 1
                    text_node.append([str(i[0]),str(secondDic[key])])
                    text_edge.append([ str(root_index) , str(i[0]) , str(key) ])


        tree = self.ensemble[index]
        text_node=[]
        text_edge=[]
        i=[1]
        export_graphviz(tree,i[0])
        #print('treenode,treeedge',text_node,text_edge)
        dot = Digraph()
        for line in text_node:
            dot.node(line[0],line[1])
        for line in text_edge:
            dot.edge(line[0],line[1],line[2])

        dot.view()
    def predict(self,X_test):
        return np.array([self._predict(test) for test in X_test])
        
    def _predict(self,test):
        """对单条测试集进行预测"""
        def __predict(tree,test):
            feature = next(iter(tree))
            #print('tree',tree,iter(tree),next(iter(tree)))
            secondDic = tree[feature]
            content = test[feature]
            for key in secondDic:
                if eval(str(content)+key):
                    if type(secondDic[key]) == dict :
                        return __predict(secondDic[key],test)
                    else:
                        return secondDic[key]

        assert len(self.ensemble) != 0,"fit before predict"
        res = 0
        for i in range(len(self.ensemble)):
            tree = self.ensemble[i]
            res_temp = __predict(tree,test)
            res += res_temp*self.eta
        res+=np.log(6.0/4)
        return 1/(1+np.exp(-res))
def main():
    #准备数据
    np.random.seed(28)
    x = np.random.randn(10, 2) * 5
    y = np.array([1] * 6 + [0] * 4).astype(np.int)
    y_true = y
    n = 4
    
    #手动实现gbdt二分类
    reg = Try_GBDT(num_round=n,y_true=y_true) 
    haty,y_train,newtree=reg.fit(x)
    #reg.draw_one_tree(3)
    print('gbdtself_predict',reg.predict(x))
    
    #gbdt官方实现
    algox = GradientBoostingClassifier(n_estimators=n, max_depth=1)
    algox.fit(x, y_true)
    from sklearn import tree
    import pydotplus
    j=0
    for i in algox.estimators_: 
        dot_data = tree.export_graphviz(decision_tree=i[0], out_file=None)
        graph = pydotplus.graph_from_dot_data(dot_data)
        #print(i[0].predict(x))
        graph.write_png('class%s.png'%(j))
        j+=1
    print('gbdtoffi_predict',algox.predict_proba(x))
main()

`这里 的大部分代码来自：xgboost手动实现https://blog.csdn.net/weixin_44264662/article/details/100896521