# XGBoost学习（六）：输出特征重要性以及筛选特征

6 篇文章 26 订阅

## XGBoost输出特征重要性以及筛选特征

### 2，绘制特征重要性

print(model.feature_importances_)


# plot
pyplot.bar(range(len(model.feature_importances_)), model.feature_importances_)
pyplot.show()


# plot feature importance manually
from numpy import loadtxt
from xgboost import XGBClassifier
from matplotlib import pyplot
from sklearn.datasets import load_iris
# split data into X and y
X = dataset.data
y = dataset.target
# fit model no training data
model = XGBClassifier()
model.fit(X, y)
# feature importance
print(model.feature_importances_)
# plot
pyplot.bar(range(len(model.feature_importances_)), model.feature_importances_)
pyplot.show()


[0.17941953 0.11345647 0.41556728 0.29155672]

这种绘制的缺点在于，只显示了特征重要性而没有排序，可以在绘制之前对特征重要性得分进行排序。
通过内建的绘制函数进行特征重要性得分排序后的绘制，这个函数就是plot_importance()，示例如下：

# plot feature importance manually
from numpy import loadtxt
from xgboost import XGBClassifier
from matplotlib import pyplot
from sklearn.datasets import load_iris
from xgboost import plot_importance

# split data into X and y
X = dataset.data
y = dataset.target
# fit model no training data
model = XGBClassifier()
model.fit(X, y)
# feature importance
print(model.feature_importances_)
# plot feature importance

plot_importance(model)
pyplot.show()


### 3，根据Xgboost特征重要性得分进行特征选择

# select features using threshold
selection = SelectFromModel(model, threshold=thresh, prefit=True)
select_X_train = selection.transform(X_train)
# train model
selection_model = XGBClassifier()
selection_model.fit(select_X_train, y_train)
# eval model
select_X_test = selection.transform(X_test)
y_pred = selection_model.predict(select_X_test)


# plot feature importance manually
import numpy as np
from xgboost import XGBClassifier
from matplotlib import pyplot
from sklearn.datasets import load_iris
from xgboost import plot_importance
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import SelectFromModel

# split data into X and y
X = dataset.data
y = dataset.target

# split data into train and test sets
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.33,random_state=7)

# fit model no training data
model = XGBClassifier()
model.fit(X_train, y_train)
# feature importance
print(model.feature_importances_)

# make predictions for test data and evaluate
y_pred = model.predict(X_test)
predictions = [round(value) for value in y_pred]
accuracy = accuracy_score(y_test,predictions)
print("Accuracy:%.2f%%"%(accuracy*100.0))

#fit model using each importance as a threshold
thresholds = np.sort(model.feature_importances_)
for thresh in thresholds:
# select features using threshold
selection = SelectFromModel(model,threshold=thresh,prefit=True )
select_X_train = selection.transform(X_train)
# train model
selection_model = XGBClassifier()
selection_model.fit(select_X_train, y_train)
# eval model
select_X_test = selection.transform(X_test)
y_pred = selection_model.predict(select_X_test)
predictions = [round(value) for value in y_pred]
accuracy = accuracy_score(y_test,predictions)
print("Thresh=%.3f, n=%d, Accuracy: %.2f%%" % (thresh, select_X_train.shape[1], accuracy * 100.0))


[0.20993228 0.09029345 0.54176074 0.15801354]
Accuracy:92.00%
Thresh=0.090, n=4, Accuracy: 92.00%
Thresh=0.158, n=3, Accuracy: 92.00%
Thresh=0.210, n=2, Accuracy: 86.00%
Thresh=0.542, n=1, Accuracy: 90.00%

### 4，网格搜索

from sklearn.model_selection import GridSearchCV
tuned_parameters= [{'n_estimators':[100,200,500],
'max_depth':[3,5,7], ##range(3,10,2)
'learning_rate':[0.5, 1.0],
'subsample':[0.75,0.8,0.85,0.9]
}]
tuned_parameters= [{'n_estimators':[100,200,500,1000]
}]
clf = GridSearchCV(XGBClassifier(silent=0,nthread=4,learning_rate= 0.5,min_child_weight=1, max_depth=3,gamma=0,subsample=1,colsample_bytree=1,reg_lambda=1,seed=1000), param_grid=tuned_parameters,scoring='roc_auc',n_jobs=4,iid=False,cv=5)
clf.fit(X_train, y_train)
##clf.grid_scores_, clf.best_params_, clf.best_score_
print(clf.best_params_)
y_true, y_pred = y_test, clf.predict(X_test)
print"Accuracy : %.4g" % metrics.accuracy_score(y_true, y_pred)
y_proba=clf.predict_proba(X_test)[:,1]
print "AUC Score (Train): %f" % metrics.roc_auc_score(y_true, y_proba)


from sklearn.model_selection import GridSearchCV
parameters= [{'learning_rate':[0.01,0.1,0.3],'n_estimators':[1000,1200,1500,2000,2500]}]
clf = GridSearchCV(XGBClassifier(
max_depth=3,
min_child_weight=1,
gamma=0.5,
subsample=0.6,
colsample_bytree=0.6,
objective= 'binary:logistic', #逻辑回归损失函数
scale_pos_weight=1,
reg_alpha=0,
reg_lambda=1,
seed=27
),
param_grid=parameters,scoring='roc_auc')
clf.fit(X_train, y_train)
print(clf.best_params_)
y_pre= clf.predict(X_test)
y_pro= clf.predict_proba(X_test)[:,1]
print "AUC Score : %f" % metrics.roc_auc_score(y_test, y_pro)
print"Accuracy : %.4g" % metrics.accuracy_score(y_test, y_pre)


import pandas as pd
import matplotlib.pylab as plt
feat_imp = pd.Series(clf.booster().get_fscore()).sort_values(ascending=False)
feat_imp.plot(kind='bar', title='Feature Importances')
plt.ylabel('Feature Importance Score')
plt.show()


## 补充：关于随机种子——random_state

random_state是一个随机种子，是在任意带有随机性的类或者函数里作为参数来控制随机模式。random_state取某一个值的时候，也就确定了一种规则。
random_state可以用于很多函数，比如训练集测试集的划分；构建决策树；构建随机森林

### 2，构建决策树的函数

clf = tree.DecisionTreeClassifier(criterion="entropy",random_state=30,splitter="random")


### 3，构建随机森林

clf = RandomForestClassifier(random_state=0)


### 4，总结

• 41
点赞
• 319
收藏
觉得还不错? 一键收藏
• 4
评论
05-28 371
04-13 2万+
05-29 4万+
08-26 1万+
05-13 3682

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。