机器学习实验（六）：用特征值衰减正则化方法进行深度学习实验_1

声明：版权所有，转载请联系作者并注明出处  http://blog.csdn.net/u013719780?viewmode=contents

#This code implements a light version of the Eigenvalue Decay regularizer for
#the Keras deep learning library, approximating the dominant eigenvalue by a
#soft function given by the power method. (It only works with Theano backend)




import numpy as np
from keras import backend as K
from keras.regularizers import Regularizer


class EigenvalueRegularizer(Regularizer):
    """This class implements the Eigenvalue Decay regularizer.
    
    Args:
        The constant that controls the regularization on the current layer
        ( see Section 3 of https://arxiv.org/abs/1604.06985 )
    Returns:
        The regularized loss (for the training data) and
        the original loss (for the validation data).
        
    """
    def __init__(self, k):
        self.k = k
        self.uses_learning_phase = True


    def set_param(self, p):
        self.p = p


    def __call__(self, loss):
        power = 9  # number of iterations of the power method
        W = self.p
        WW = K.dot(K.transpose(W), W)
        dim1, dim2 = K.eval(K.shape(WW))
        k = self.k
        o = np.ones(dim1)  # initial values for the dominant eigenvector

        # power method for approximating the dominant eigenvector:
        domin_eigenvect = K.dot(WW, o)
        for n in range(power - 1):
            domin_eigenvect = K.dot(WW, domin_eigenvect)    
        
        WWd = K.dot(WW, domin_eigenvect)
        domin_eigenval = K.dot(WWd, domin_eigenvect) / K.dot(domin_eigenvect, domin_eigenvect)  # the corresponding dominant eigenvalue
        regularized_loss = loss + (domin_eigenval ** 0.5) * self.k  # multiplied by the given regularization gain
        return K.in_train_phase(regularized_loss, loss)
    

    def get_config(self):
        return {"name": self.__class__.__name__,
"k": self.k}

Using Theano backend.