list @size 验证_【ML】xgboostAPI-网格搜索-交叉验证

先看下xgb原生api训练的参数：

 def train(params, dtrain, num_boost_round=10, evals=(), obj=None, feval=None,
              maximize=False, early_stopping_rounds=None, evals_result=None,
              verbose_eval=True, xgb_model=None, callbacks=None)

• 目前关注以下几个参数
  • params，模型参数，见下面
  • dtrain，训练数据，DMatrix格式
  • num_boost_round，迭代次数
  • evals，验证集
  • early_stopping_rounds，提前停止
  • evals_result，保存训练结果

导包

import numpy as np
import xgboost as xgb

from sklearn.datasets import load_breast_cancer, load_boston
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

分类

# data

[12] train-error:0.00000 test-error:0.06140
[13] train-error:0.00000 test-error:0.05263
Stopping. Best iteration:
[3] train-error:0.00000 test-error:0.05263
error= 0.05

回归：

data = load_boston()
x_data = data.data
y_data = data.target                                
x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=0.2, random_state=999)

dtrain = xgb.DMatrix(x_train, y_train)
dtest = xgb.DMatrix(x_test, y_test)


# params
params = {
    'objective': 'reg:squarederror',                 # 回归，平方误差目标函数
    'max_depth': 2,
    'eta': 1
}
num_round = 50
watch_list = [(dtrain, 'train'), (dtest, 'test')]
evals_result = {}                                   # 保存训练结果

# train
bst = xgb.train(params, 
                dtrain, 
                num_round, 
                evals=watch_list, 
                early_stopping_rounds=10, 
                evals_result=evals_result)

# 回归输出的是直接的预测值，可以直接比较
preds = bst.predict(dtest)
labels = dtest.get_label()

# rmse
print(np.sqrt(mean_squared_error(preds,labels)))

[16] train-rmse:2.20856 test-rmse:3.68877
[17] train-rmse:2.15026 test-rmse:3.62916
[18] train-rmse:2.09039 test-rmse:3.62334
Stopping. Best iteration:
[8] train-rmse:2.78789 test-rmse:3.56546
rmse: 3.623343

多分类

# 模拟数据：构造100*10的样本特征，一共有4个类别
np.random.seed(1994)

kRows = 500
kCols = 10
kClasses = 4                    


x_data = np.random.randn(kRows, kCols)
y_data = np.random.randint(0, 4, size=kRows)

x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=0.2, random_state=999)

dtrain = xgb.DMatrix(x_train, y_train)
dtest = xgb.DMatrix(x_test, y_test)


params = {
    'objective': 'multi:softmax',                 # 多分类
    'num_class': 4,                               # 必须指定类别个数
    'max_depth': 2
}


num_round = 50
watch_list = [(dtrain, 'train'), (dtest, 'test')]
evals_result = {}                                   # 保存训练结果

# train
bst = xgb.train(params, 
                dtrain, 
                num_round, 
                evals=watch_list, 
                early_stopping_rounds=10, 
                evals_result=evals_result)

# 多分类输出的直接是类别，可以直接比较
preds = bst.predict(dtest)
labels = dtest.get_label()

error = sum([1 for i in range(len(preds)) if preds[i] != labels[i]]) / float(len(preds))

print(f'error={error: .2f}'

123
[11] train-merror:0.38000 test-merror:0.74000
[12] train-merror:0.36500 test-merror:0.74000
[13] train-merror:0.35500 test-merror:0.75000
Stopping. Best iteration:
[3] train-merror:0.47750 test-merror:0.67000
error= 0.70

训练的时候指定了early_stopping_rounds，那么在预测的时候可以指定只用最好的迭代次数来预测：

preds = bst.predict(dtest, ntree_limit=bst.best_iteration)

调参

常用的模型参数如下：

• num_boost_round：模型迭代次数，即训练了多少棵树，越大越容易过拟合，默认100
• max_depth：一棵树的最大深度，越大越容易过拟合，默认6
• min_child_weight：子节点最小的权重分数，比如当前节点分裂了两个子节点，如果子节点的得分小于该值，就停止分裂。回归任务表示叶子节点最少的样本数。越大越容易防止过拟合，默认1
• subsample：是一个分数，表示每次训练一棵树选取多少比例的样本，越大越容易过拟合，默认1
• colsample_bytree：是一个分数，表示训练一棵树选多少比列的特征，越大越容易过拟合，默认1
• gamma：正则项控制叶子节点的系数，即当分裂节点的时候，使损失下降多少才可以分裂节点，越大越容易防止过拟合，默认0，
• lambda：正则项控制分数w的系数，l2正则系数，越大越容易防止过拟合，默认1，
• eta：学习率，shrinkage思想，每棵树的系数，加权值， 越大越容易防止过拟合，默认0.3

• 还有一些配置项参数，就不涉及调参了，booster指定弱分类器模型，verbosity打印日志级别等等
• 更多可参考官网：https://xgboost.readthedocs.io/en/latest/parameter.html#

一般调参步骤：

1. 先确定num_boost_round=100

2. 再调max_depth，min_child_weight，subsample，colsample_bytree

3. 再调gamma，lambda

4. 再调eta和num_boost_round

依次调参的时候，都可以用上一步调好的参数训练，用验证集看看还有什么问题，然后适当来调节下一步参数的范围？

然后xgboost原生API是不支持sklearn的网格搜索的，要么自己写循环调用原生的cv验证，要么写一个自定义类用sklearn的网格搜索。

# 自定义支持sklearn的评估器
class MyXGBoostEstimator:
    def __init__(self, **params):
        self.params = params
        if 'num_boost_round' in self.params:
            self.num_boost_round = self.params['num_boost_round']
        # 默认是回归，如果是分类，要修改objective
        self.params.update({'silent':1, 'objective':'reg:squarederror', 'seed':0})
        
    def fit(self, x_train, y_train):
        dtrain = xgb.DMatrix(x_train, y_train)
        self.bst = xgb.train(params=self.params, dtrain=dtrain, num_boost_round=self.num_boost_round)
        
    def predict(self, x_pred):
        dpred = xgb.DMatrix(x_pred)
        return self.bst.predict(dpred)
    
    def kfold(self, x_train, y_train, nfold=5):
        dtrain = xgb.DMatrix(x_train, y_train)
        
        cv_rounds = xgb.cv(params=self.params, dtrain=dtrain, num_boost_round=self.num_boost_round,
                           nfold=nfold, early_stopping_rounds=10)
        
        return cv_rounds.iloc[-1,:]

    def get_params(self, deep=True):
        return self.params
    
    def set_params(self, **params):
        self.params.update(params)
        return self

必须要实现上述的方法，fit，predict是为了训练预测，kfold是进行cv验证，get和set是进行网格搜索的时候设置不同参数

# 评估函数：RMSE
def score_fn(y_pred, y_true):
    return np.sqrt(mean_squared_error(y_pred, y_true))

cv_score_fn = make_scorer(score_fn, greater_is_better=False)

进行网格搜索：

训练：max_depth，min_child_weight，subsample，colsample_bytree

param_grid = {'max_depth': list([5, 6, 7]), 
             'min_child_weight': list([1, 5, 9]),
             'subsample': list([0.8, 1]),
             'colsample_bytree': list([0.8, 1])}


model = MyXGBoostEstimator(num_boost_round=100)

grid = GridSearchCV(model, param_grid=param_grid, cv=3, scoring=cv_score_fn)

grid.fit(x_train, y_train)

print(grid.best_params_)

{'colsample_bytree': 1,
'max_depth': 7,
'min_child_weight': 5,
'subsample': 0.8}

训练：gamma，lambda，

# 将已经训练好的参数设置为默认参数
params = {'num_boost_round': 100,
         'colsample_bytree': 1,
         'max_depth': 7,
         'min_child_weight': 5,
         'subsample': 0.8}

# 这两个参数，调的话要看前面训练loss的下降是情况，可以先拿前面的参数去训一下，如果过拟合严重，再来调这个
param_grid = {'gamma': list([0, 0.1, 0.5]), 
             'lambda': list([1, 1.5])}

model = MyXGBoostEstimator(**params)

grid = GridSearchCV(model, param_grid=param_grid, cv=3, scoring=cv_score_fn)

grid.fit(x_train, y_train)

print(grid.best_params_)

{'gamma': 0, 'lambda': 1}

• 可以看到这里参数基本调出来还是默认

训练：eta, num_boost_round

params = {'colsample_bytree': 1,
         'max_depth': 7,
         'min_child_weight': 5,
         'subsample': 0.8}

etas = [0.3, 0.5, 1]
num_boost_rounds = [15, 50, 100]
nfold = 3

best_eta, best_round = 0, 0
best_score = 10000000


for eta in etas:
    for num_boost_round in num_boost_rounds:
        params.update({'eta': eta})
        dtrain = xgb.DMatrix(x_train, y_train)
        
        cv_rounds = xgb.cv(params=params, dtrain=dtrain, num_boost_round=num_boost_round,
                           nfold=nfold, early_stopping_rounds=10)
        
        # cv返回的是num_boost_round轮评估指标的train-mean，train-std，test-mean，test-std
        # 这里取最后一轮的测试集评估指标均值作为score
        score = cv_rounds.iloc[-1,-2]
        
        if score < best_score:
            best_score = score
            best_eta = eta
            best_round = num_boost_round
            
            
print(f'eta: {best_eta}, num_boost_round: {best_round}')

eta: 0.3, num_boost_round: 100

参数得到后，用最优参数训练看看结果：

params = {'colsample_bytree': 1,
         'max_depth': 7,
         'min_child_weight': 5,
         'subsample': 0.8,
         'eta': 0.3,
         'gamma': 0, 
         'lambda': 1}

dtrain = xgb.DMatrix(x_train, y_train)
dtest = xgb.DMatrix(x_test, y_test)

watch_list = [(dtrain, 'train'), (dtest, 'test')]
evals_result = {}                                   # 保存训练结果

# train
bst = xgb.train(params=params, dtrain=dtrain, num_boost_round=100,
                evals=watch_list, 
                early_stopping_rounds=10, 
                evals_result=evals_result)

[25] train-rmse:0.97110 test-rmse:2.83420
[26] train-rmse:0.95397 test-rmse:2.80972
[27] train-rmse:0.88680 test-rmse:2.82232
Stopping. Best iteration:
[17] train-rmse:1.30593 test-rmse:2.77281

• 相对于之前的回归，loss有明显的下降。