T r e e D e p t h C h i l d W e i g h t − 1 TreeDepthChildWeight-1 TreeDepthChildWeight−1
XGBoost Parameter Tuning for Otto Dataset
第二步:调整树的参数:max_depth & min_child_weight
(粗调,参数的步长为2;下一步是在粗调最佳参数周围,将步长降为1,进行精细调整)
导入所需工具包
from xgboost import XGBClassifier
import xgboost as xgb
import pandas as pd
import numpy as np
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from matplotlib import pyplot
import seaborn as sns
%matplotlib inline
读取数据
# path to where the data lies
dpath = './data/'
train = pd.read_csv(dpath +"Otto_train.csv")
#train.head()
Variable Identification
选择该数据集是因为的数据特征单一,我们可以在特征工程方面少做些工作,集中精力放在参数调优上
Target 分布,看看各类样本分布是否均衡
sns.countplot(train.target);
pyplot.xlabel('target');
pyplot.ylabel('Number of occurrences');
每类样本分布不是很均匀,所以交叉验证时也考虑各类样本按比例抽取
# drop ids and get labels
y_train = train['target']
y_train = y_train.map(lambda s: s[6:])
y_train = y_train.map(lambda s: int(s)-1)
train = train.drop(["id", "target"], axis=1)
X_train = np.array(train)
各类样本不均衡,交叉验证是采用StratifiedKFold,在每折采样时各类样本按比例采样
# prepare cross validation
kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)
第一轮参数调整得到的n_estimators最优值(699),其余参数继续默认值
用交叉验证评价模型性能时,用scoring参数定义评价指标。评价指标是越高越好,因此用一些损失函数当评价指标时,需要再加负号,如neg_log_loss,neg_mean_squared_error 详见sklearn文档:http://scikitlearn.org/stable/modules/model_evaluation.html#log-loss
#max_depth 建议3-10, min_child_weight=1/sqrt(ratio_rare_event) =5.5
max_depth = range(3,10,2)
min_child_weight = range(1,6,2)
param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)
param_test2_1
xgb2_1 = XGBClassifier(
learning_rate =0.1,
n_estimators=2, #第一轮参数调整得到的n_estimators最优值
max_depth=5,
min_child_weight=1,
gamma=0,
subsample=0.3,
colsample_bytree=0.8,
colsample_bylevel = 0.7,
objective= 'multi:softprob',
seed=3)
gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)
gsearch2_1.fit(X_train , y_train)
gsearch2_1.grid_scores_, gsearch2_1.best_params_, gsearch2_1.best_score_
gsearch2_1.cv_results_
# summarize results
print("Best: %f using %s" % (gsearch2_1.best_score_, gsearch2_1.best_params_))
test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]
test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]
train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]
train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]
pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')
# plot results
test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))
train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))
for i, value in enumerate(max_depth):
pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:' + str(value))
#for i, value in enumerate(min_child_weight):
# pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:' + str(value))
pyplot.legend()
pyplot.xlabel( 'max_depth' )
pyplot.ylabel( 'Log Loss' )
pyplot.savefig('max_depth_vs_min_child_weght_1.png' )