keras中自定义验证集的性能评估（ROC,AUC）

在keras中自带的性能评估有准确性以及loss，当需要以auc作为评价验证集的好坏时，就得自己写个评价函数了：

[python] view plain copy

1. from sklearn.metrics import roc_auc_score  

[python] view plain copy

1. from keras import backend as K  

[python] view plain copy

1. # AUC for a binary classifier  
2. def auc(y_true, y_pred):  
3.     ptas = tf.stack([binary_PTA(y_true,y_pred,k) for k in np.linspace(0, 1, 1000)],axis=0)  
4.     pfas = tf.stack([binary_PFA(y_true,y_pred,k) for k in np.linspace(0, 1, 1000)],axis=0)  
5.     pfas = tf.concat([tf.ones((1,)) ,pfas],axis=0)  
6.     binSizes = -(pfas[1:]-pfas[:-1])  
7.     s = ptas*binSizes  
8.     return K.sum(s, axis=0)  
9. #-----------------------------------------------------------------------------------------------------------------------------------------------------  
10. # PFA, prob false alert for binary classifier  
11. def binary_PFA(y_true, y_pred, threshold=K.variable(value=0.5)):  
12.     y_pred = K.cast(y_pred >= threshold, 'float32')  
13.     # N = total number of negative labels  
14.     N = K.sum(1 - y_true)  
15.     # FP = total number of false alerts, alerts from the negative class labels  
16.     FP = K.sum(y_pred - y_pred * y_true)  
17.     return FP/N  
18. #-----------------------------------------------------------------------------------------------------------------------------------------------------  
19. # P_TA prob true alerts for binary classifier  
20. def binary_PTA(y_true, y_pred, threshold=K.variable(value=0.5)):  
21.     y_pred = K.cast(y_pred >= threshold, 'float32')  
22.     # P = total number of positive labels  
23.     P = K.sum(y_true)  
24.     # TP = total number of correct alerts, alerts from the positive class labels  
25.     TP = K.sum(y_pred * y_true)  
26.     return TP/P  
27.   
28. #接着在模型的compile中设置metrics  
29. #如下例子，我用的是RNN做分类  

from keras.models import Sequential
from keras.layers import Dense, Dropout
import keras
from keras.layers import GRU

model = Sequential()
model.add(keras.layers.core.Masking(mask_value=0., input_shape=(max_lenth, max_features))) #masking用于变长序列输入
model.add(GRU(units=n_hidden_units,activation='selu',kernel_initializer='orthogonal', recurrent_initializer='orthogonal',
              bias_initializer='zeros', kernel_regularizer=regularizers.l2(0.01), recurrent_regularizer=regularizers.l2(0.01),
              bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None,
              bias_constraint=None, dropout=0.5, recurrent_dropout=0.0, implementation=1, return_sequences=False,
              return_state=False, go_backwards=False, stateful=False, unroll=False))   
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=[auc])  #写入自定义评价函数

[python] view plain copy

1.   

接下来就自己作预测了...

方法二：

[python] view plain copy

1. from sklearn.metrics import roc_auc_score  
2. import keras  
3. class RocAucMetricCallback(keras.callbacks.Callback):  
4.     def __init__(self, predict_batch_size=1024, include_on_batch=False):  
5.         super(RocAucMetricCallback, self).__init__()  
6.         self.predict_batch_size=predict_batch_size  
7.         self.include_on_batch=include_on_batch  
8.   
9.     def on_batch_begin(self, batch, logs={}):  
10.         pass  
11.   
12.     def on_batch_end(self, batch, logs={}):  
13.         if(self.include_on_batch):  
14.             logs['roc_auc_val']=float('-inf')  
15.             if(self.validation_data):  
16.                 logs['roc_auc_val']=roc_auc_score(self.validation_data[1],   
17.                                                   self.model.predict(self.validation_data[0],  
18.                                                                      batch_size=self.predict_batch_size))  
19.   
20.     def on_train_begin(self, logs={}):  
21.         if not ('roc_auc_val' in self.params['metrics']):  
22.             self.params['metrics'].append('roc_auc_val')  
23.   
24.     def on_train_end(self, logs={}):  
25.         pass  
26.   
27.     def on_epoch_begin(self, epoch, logs={}):  
28.         pass  
29.   
30.     def on_epoch_end(self, epoch, logs={}):  
31.         logs['roc_auc_val']=float('-inf')  
32.         if(self.validation_data):  
33.             logs['roc_auc_val']=roc_auc_score(self.validation_data[1],   
34.                                               self.model.predict(self.validation_data[0],  
35.                                                                  batch_size=self.predict_batch_size))  
36.   
37.   
38. import numpy as np  
39. import tensorflow as tf  
40. from keras.models import Sequential  
41. from keras.layers import Dense, Dropout  
42. from keras.layers import GRU  
43. import keras  
44. from keras.callbacks import EarlyStopping  
45. from sklearn.metrics import roc_auc_score  
46. from keras import metrics  
47.   
48. cb = [  
49.     my_callbacks.RocAucMetricCallback(), # include it before EarlyStopping!  
50.     EarlyStopping(monitor='roc_auc_val',patience=300, verbose=2,mode='max')  
51. ]  
52. model = Sequential()  
53. model.add(keras.layers.core.Masking(mask_value=0., input_shape=(max_lenth, max_features)))  
54. # model.add(Embedding(input_dim=max_features+1, output_dim=64,mask_zero=True))  
55. model.add(GRU(units=n_hidden_units,activation='selu',kernel_initializer='orthogonal', recurrent_initializer='orthogonal',  
56.               bias_initializer='zeros', kernel_regularizer=regularizers.l2(0.01), recurrent_regularizer=regularizers.l2(0.01),  
57.               bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None,  
58.               bias_constraint=None, dropout=0.5, recurrent_dropout=0.0, implementation=1, return_sequences=False,  
59.               return_state=False, go_backwards=False, stateful=False, unroll=False))   #input_shape=(max_lenth, max_features),  
60. model.add(Dropout(0.5))  
61. model.add(Dense(1, activation='sigmoid'))  
62.   
63. model.compile(loss='binary_crossentropy',  
64.               optimizer='adam',  
65.               metrics=[auc]) #这里就可以写其他评估标准  
66. model.fit(x_train, y_train, batch_size=train_batch_size, epochs=training_iters, verbose=2,  
67.           callbacks=cb,validation_split=0.2,  
68.           shuffle=True, class_weight=None, sample_weight=None, initial_epoch=0)  

亲测有效