分类模型评估指标

导入数据

from sklearn.datasets import load_iris
iris=load_iris()

拆分数据

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.3, random_state=666)

构建决策树模型

from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier()
clf.fit(x_train,y_train)

DecisionTreeClassifier(class_weight=None, criterion=‘gini’, max_depth=None,
max_features=None, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, presort=False,
random_state=None, splitter=‘best’)

模型预测

y_pred=clf.predict(x_test)

模型评估–混淆矩阵

from sklearn.metrics import classification_report
a=classification_report(y_train,
                       clf.predict(x_train),
                       digits=3,#小数点后保留位数
                       labels=[2,1,0],#类别的排序
                       target_names=['第二类','第一类','第0类'],#类别名称
                       output_dict=False,#不以字典形式输出
                      )

输出训练集上的混淆举证

from sklearn.metrics import confusion_matrix
cm=confusion_matrix(y_train,clf.predict(x_train))
cm

array([[38, 0, 0],
[ 0, 32, 0],
[ 0, 0, 35]], dtype=int64)

自定义类别顺序输出混淆矩阵

confusion_matrix(y_train,clf.predict(x_train,labels=[2,1,0]))

用热力图形式展示混淆举证

%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
sns.heatmap(cm,cmap=sns.color_palette("Blues"),annot=True)

roc曲线

#roc曲线
from sklearn.metrics import roc_curve
import numpy as np

#录入数据
y_true=np.array([1,1,0,1,1,0,0,0,1,0])
y_score=np.array([0.9,0.8,0.7,0.6,0.55,0.54,0.53,0.51,0.50,0.40])

# 调用roc_curve 求出fpr与tpr
fpr,tpr,thresholds=roc_curve(y_true,y_score)
print(fpr,tpr,thresholds,sep='\n')

[0. 0. 0. 0.2 0.2 0.8 0.8 1. ]
[0. 0.2 0.4 0.4 0.8 0.8 1. 1. ]
[1.9 0.9 0.8 0.7 0.55 0.51 0.5 0.4 ]

修改参数drop_intermediate=Fals

#drop_intermediate=False
#drop_intermediate=False
fpr1,tpr1,thresholds1=roc_curve(y_true,y_score,drop_intermediate=False)
print(fpr1,tpr1,thresholds1,sep='\n')

[0. 0. 0. 0.2 0.2 0.2 0.4 0.6 0.8 0.8 1. ]
[0. 0.2 0.4 0.4 0.6 0.8 0.8 0.8 0.8 1. 1. ]
[1.9 0.9 0.8 0.7 0.6 0.55 0.54 0.53 0.51 0.5 0.4 ]

结果可视化

import matplotlib.pyplot as plt
plt.plot(fpr1,tpr1,'g*-')
plt.plot([0,1],[0,1])
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.title('ROC')

sklearn.metircs.roc_auc_score计算roc曲线下的面积

# sklearn.metircs.roc_auc_score计算roc曲线下的面积
from sklearn.metrics import roc_auc_score
roc_auc_score(y_true,y_score)

0.76

交叉验证（cv）–cross validation

sklearn.model_selection.cross_val_score()
sklearn.model_selection.cross_validate()
sklearn.model_selection.cross_val_predict()

简单交叉验证、s折交叉验证（s-fold cross validation）、留一交叉验证（训练数量少的时候）

sklearn,model_slection中最常用的有以下三个
cross_val_score
cross_validate
cross_val_predict

from sklearn import model_selection
from sklearn.datasets import load_boston
boston=load_boston()

from sklearn.linear_model import LinearRegression
reg=LinearRegression()
from sklearn.model_selection import cross_val_score
scores=cross_val_score(reg,boston.data,boston.target,cv=10)
scores

array([ 0.73376082, 0.4730725 , -1.00631454, 0.64113984, 0.54766046,
0.73640292, 0.37828386, -0.12922703, -0.76843243, 0.4189435 ])

scores.mean(),scores.std()

scores=cross_val_score(reg,boston.data,boston.target,scoring='explained_variance',cv=10)
scores

array([ 0.74784412, 0.5381936 , -0.80757662, 0.66844779, 0.5586898 ,
0.74128804, 0.41981565, -0.11666214, -0.44561819, 0.42197365])

保证案例顺序的随机性

如果非随机，则会对模型验证带来严重影响

对数据随机重排列，保证拆分的均匀性

# 保证案例顺序的随机性
# 如果非随机，则会对模型验证带来严重影响
# 对数据随机重排列，保证拆分的均匀性
import numpy as np
X,y=boston.data,boston.target
indices=np.arange(y.shape[0])
np.random.shuffle(indices)
X,y=X[indices],y[indices]

# 结果变好
from sklearn.linear_model import LinearRegression
reg=LinearRegression()
from sklearn.model_selection import cross_val_score
scores=cross_val_score(reg,boston.data,boston.target,cv=10)
scores

array([ 0.73376082, 0.4730725 , -1.00631454, 0.64113984, 0.54766046,
0.73640292, 0.37828386, -0.12922703, -0.76843243, 0.4189435 ])

同时使用多个指标

from sklearn.model_selection import cross_validate
scoring=['r2','explained_variance']
scores=cross_validate(reg,X,y,cv=10,scoring=scoring,return_train_score=False)
scores

{‘fit_time’: array([0.00300241, 0.00201869, 0.00198293, 0.00200033, 0.00299835,
0.00200009, 0.00099754, 0.00099993, 0.00100064, 0.00099754]),
‘score_time’: array([0.00255346, 0.00198126, 0.00200129, 0.0030036 , 0.00099969,
0.0020082 , 0.00199842, 0.0010004 , 0.00199986, 0.00300241]),
‘test_r2’: array([0.81991125, 0.62882946, 0.76762236, 0.68529018, 0.8538444 ,
0.46967575, 0.75151397, 0.87519807, 0.59658192, 0.61027392]),
‘test_explained_variance’: array([0.86204884, 0.63050762, 0.7676528 , 0.68529892, 0.86176216,
0.48324984, 0.75693009, 0.87521126, 0.62049691, 0.61305254])}

scores['test_r2'].mean()

0.7058741277117355

from sklearn.model_selection import cross_val_predict
pred=cross_val_predict(reg,X,y,cv=10)
pred[:10]
from sklearn.metrics import r2_score
r2_score(y,pred)

网格搜索

from sklearn.model_selection import GridSearchCV

from sklearn import svm
from sklearn import datasets
from sklearn.model_selection import GridSearchCV
import pandas as pd
iris=datasets.load_iris()
parameters={'kernel':('linear','rbf'),'C':[1,10]}
svc=svm.SVC(probability=True)
clf=GridSearchCV(svc,parameters)
clf.fit(iris.data,iris.target)

from sklearn.svm import SVC
# 显示所有拟合模型的参数设定
pd.DataFrame(clf.cv_results_)

from sklearn.model_selection import RandomizedSearchCV
import scipy.stats as stats
from sklearn import datasets

from sklearn.model_selection import RandomizedSearchCV
import pandas as pd
from sklearn.svm import SVC
iris=datasets.load_iris()
parameters={'kernel':('linear','rbf'),
           'C':stats.expon(scale=100),
           'gamma':stats.expon(scale=1),
           'class_weight':('balanced',None)}
svc=SVC()
clf=RandomizedSearchCV(svc,parameters)
clf.fit(iris.data,iris.target)
pd.DataFrame(clf.cv_results_)