贷款逾期（4）--五个模型整合

模型评估

import pickle
from matplotlib import pyplot as plt
from sklearn.externals import joblib
from sklearn.metrics import accuracy_score, recall_score, f1_score, roc_auc_score, roc_curve

path = "E:/mypython/Machine_learning_GoGoGo/"
"""=====================================================================================================================
1 读取特征
"""
print("0 读取特征")
f = open(path + 'feature/feature_V1.pkl', 'rb')
train, test, y_train,y_test= pickle.load(f)
f.close()

"""=====================================================================================================================
2 读取模型
"""
print("1 读取模型")
SVM_linear = joblib.load( path + "model/SVM_linear.pkl")

SVM_rbf = joblib.load( path + "model/SVM_rbf.pkl")
SVM_sigmoid = joblib.load( path + "model/SVM_sigmoid.pkl")
lg_120 = joblib.load( path + "model/lg_120.pkl")
DT = joblib.load( path + "model/DT.pkl")
xgb_sklearn = joblib.load( path + "model/xgb_sklearn.pkl")
lgb_sklearn = joblib.load( path + "model/lgb_sklearn.pkl")
xgb = joblib.load( path + "model/xgb.pkl")
lgb = joblib.load( path + "model/lgb.pkl")




"""=====================================================================================================================
3 模型评估
"""

def model_evalua(clf, X_train, X_test, y_train, y_test,name):
    y_train_pred = clf.predict(X_train)
    y_test_pred = clf.predict(X_test)
    y_train_pred_proba = clf.predict_proba(X_train)[:, 1]
    y_test_pred_proba = clf.predict_proba(X_test)[:, 1]
    """【AUC Score】"""
    print(name+'_AUC Score')
    print(name+"_Train_AUC Score ：{:.4f}".format(roc_auc_score(y_train, y_train_pred)))
    print(name+"_Test_AUC Score ：{:.4f}".format(roc_auc_score(y_test, y_test_pred)))

    """【准确性】"""
    print(name+'_准确性：')
    print(name+'_Train_准确性：{:.4f}'.format(accuracy_score(y_train, y_train_pred)))
    print(name+'_Test_准确性：{:.4f}'.format(accuracy_score(y_test, y_test_pred)))

    """【召回率】"""
    print(name+'_召回率：')
    print(name+'_Train_召回率：{:.4f}'.format(recall_score(y_train, y_train_pred)))
    print(name+'_Test_召回率：{:.4f}'.format(recall_score(y_test, y_test_pred)))

    """【f1_score】"""
    print(name+'_f1_score：')
    print(name+'_Train_f1_score：{:.4f}'.format(f1_score(y_train, y_train_pred)))
    print(name+'_Test_f1_score：{:.4f}'.format(f1_score(y_test, y_test_pred)))

    #描绘 ROC 曲线
    fpr_tr, tpr_tr, _ = roc_curve(y_train, y_train_pred_proba)
    fpr_te, tpr_te, _ = roc_curve(y_test, y_test_pred_proba)
    # KS
    print(name+'_KS：')
    print(name+'_Train：{:.4f}'.format(max(abs((fpr_tr - tpr_tr)))))
    print(name+'_Test：{:.4f}'.format(max(abs((fpr_te - tpr_te)))))
    plt.plot(fpr_tr, tpr_tr, 'r-',
             label = name+"_Train:AUC: {:.3f} KS:{:.3f}".format(roc_auc_score(y_train, y_train_pred_proba),
                                                max(abs((fpr_tr - tpr_tr)))))
    plt.plot(fpr_te, tpr_te, 'g-',
             label=name+"_Test:AUC: {:.3f} KS:{:.3f}".format(roc_auc_score(y_test, y_test_pred_proba),
                                                       max(abs((fpr_tr - tpr_tr)))))
    plt.plot([0, 1], [0, 1], 'd--')
    plt.legend(loc='best')
    plt.title(name+"_ROC curse")
    plt.savefig(path +'picture/'+name+'.jpg')
    plt.show()
print('-------------------SVM_linear-------------------')
model_evalua(SVM_linear, train, test, y_train, y_test,'SVM_linear')



print('-------------------SVM_rbf-------------------：')
model_evalua(SVM_rbf, train, test, y_train, y_test,'SVM_rbf')

print('-------------------SVM_sigmoid-------------------：')
model_evalua(SVM_sigmoid, train, test, y_train, y_test,'SVM_sigmoid')

print('-------------------lg_120-------------------')
model_evalua(lg_120, train, test, y_train, y_test,'lg_120')

print('-------------------DT-------------------')
model_evalua(DT, train, test, y_train, y_test,'DT')

print('-------------------xgb_sklearn-------------------')
model_evalua(xgb_sklearn, train, test, y_train, y_test,'xgb_sklearn')

# print('-------------------xgb-------------------')
# model_evalua(xgb, train, test, y_train, y_test)

print('-------------------lgb_sklearn-------------------')
model_evalua(lgb_sklearn, train, test, y_train, y_test,'lgb_sklearn')
# print('-------------------lgb-------------------')
# model_evalua(lgb, train, test, y_train, y_test)

实验表格

	precision	recall_score	f1_score	KS	ROC_AUC
SVM-linear	Train_准确性：0.7878 Test_准确性：0.7442	Train_召回率：0.1683 Test_召回率：0.3377	f1_score：0.2781 f1_score：0.4160	Train：0.4519 Test：0.2590	Train_AUC Score ：0.5774 Test_AUC Score ：0.6160
SVM-rbf	Train_准确性：0.7971 Test_准确性：0.7589	Train_召回率：0.1894 Test_召回率：0.1455	f1_score：0.3119 f1_score：0.2456	Train：0.6474 Train：0.6474	AUC Score ：0.5907 AUC Score ：0.5655
SVM-sigmoid	Train_准确性：0.7265 Test_准确性：0.7092	Train_召回率：0.2809 Test_召回率：0.1584	f1_score：0.3328 f1_score：0.2272	Train：0.2216 Test：0.1235	Train_AUC Score ：0.5752 Test_AUC Score ：0.5356
逻辑回归	Train_准确性：0.4355 Train_准确性：0.4355	Train_召回率：0.6671 Test_召回率：0.7117	f1_score：0.3647 f1_score：0.4152	Train：0.0695 Test：0.0907	Train_AUC Score ：0.5142 Test_AUC Score ：0.5387
DT	Train_准确性：0.7920 Test_准确性：0.7505	Train_召回率：0.4245 Test_召回率：0.3169	f1_score：0.4978 f1_score：0.4067	Train：0.4126 Test：0.3524	AUC Score ：0.6672 Test_AUC Score ：0.6138
xgb-sklearn	Train_准确性：0.8452 Test_准确性：0.7765	Train_召回率：0.4691 Test_召回率：0.3065	Train_f1_score：0.5954 Test_f1_score：0.4252	Train：0.6167 Test：0.3763	Train_AUC Score ：0.7175 Test_AUC Score ：0.6283
lgb-sklearn	Train_准确性：1.0000 Test_准确性：0.7680	Train_召回率：1.0000 Test_召回率：0.3117	Train_f1_score：1.0000 Test_f1_score：0.4203	Train：1.0000 Test：0.3761	Train_AUC Score ：1.0000 Test_AUC Score ：0.6242

 

ROC-曲线