Keras--动态调整学习率

在训练模型时,有时我们需要随着训练的深入调整学习率,

这里介绍几种Keras常用方式

from keras.callbacks import Callback
from keras.backend as K

class LinearDecayLR(Callback):
    '''
    第一种,每个batch之后衰减一次学习率(lr)
    
    # Arguments
        min_lr: The lower bound (final value) of the learning rate.
        max_lr: The upper bound (initial value) of the learning rate.
        steps_per_epoch: 一个epoch所包含的batch数 
        epochs: Number of epochs to run training. 
        
    # Usage
        lr_decay = LinearDecayLR(min_lr=1e-5, max_lr=0.01, 
                                 steps_per_epoch=step_size_train, 
                                 epochs=20, verbose=1)
        model.fit(X_train, Y_train, callbacks=[lr_decay])
    '''
    
    def __init__(self, min_lr=1e-5, max_lr=1e-2, steps_per_epoch=None, epochs=None, verbose=0):
        super().__init__()
        
        self.min_lr = min_lr
        self.max_lr = max_lr
        self.total_iterations = steps_per_epoch * epochs # 每个epoch包含的batch数 * 总epoch数 = 总batch数
        self.iteration = 0
        self.verbose = verbose
        
    def linear_decay(self): # 一个epoch之后, 计算一次学习率
        '''Calculate the learning rate.'''
        r = self.iteration / self.total_iterations 
        return self.max_lr - (self.max_lr-self.min_lr) * r
        
    def on_train_begin(self, logs=None): # 整个训练的开始,给学习率赋最大值
        '''Initialize the learning rate to the initial value at the start of training.'''
        logs = logs or {}
        K.set_value(self.model.optimizer.lr, self.max_lr)
        
    def on_batch_end(self, epoch, logs=None): # 每个batch结束, 计算一下学习率, 
        '''Update the learning rate after each batch update'''
        logs = logs or {}
        self.iteration += 1
        K.set_value(self.model.optimizer.lr, self.linear_decay())
    
    def on_epoch_begin(self, epoch, logs=None):
        if self.verbose > 0:
            print('\nEpoch %05d: LearningRateScheduler setting learning '
                  'rate to %s.' % (epoch + 1, K.get_value(self.model.optimizer.lr)))    
    
    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        logs['lr'] = K.get_value(self.model.optimizer.lr)


class StepLR(Callback):
    '''
    第二种方式,台阶式的学习率调整,不一定是衰减,也可以是增加,
    epoch_list = [100, 200, 300]
    lr_list    = [0.5, 0.1, 0.8]
    举个例子,参数设置如上, 在epoch 0~99区域内, 学习率是 0.5, 在100~199范围内,学习率是0.1, 在200~300范围内,学习率是0.8
    注意两个列表一定要一样长, 这是我简单的实现, 不是很鲁棒.
    而且, 将 "lr"放入logs时,tensorboard是可以观察 lr 的,
    '''

    def __init__(self, lr_list, epoch_list, verbose=1):
        super().__init__()

        self.lr_list = lr_list
        self.epoch_list = epoch_list
        self.verbose = verbose

    def on_epoch_begin(self, epoch, logs=None):
        if epoch < self.epoch_list[0]:
            K.set_value(self.model.optimizer.lr, self.lr_list[0])
        elif epoch == self.epoch_list[0]:
            self.lr_list.pop(0)
            self.epoch_list.pop(0)
            K.set_value(self.model.optimizer.lr, self.lr_list[0])
        if self.verbose > 0:
            print('\nEpoch %05d: LearningRate'
                  'is %s.' % (epoch + 1, K.get_value(self.model.optimizer.lr)))

    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        logs['lr'] = K.get_value(self.model.optimizer.lr)

用法示例

epoch_list = [100, 200, 300]
lr_list    = [0.5, 0.1, 0.8]
stepLR = StepLR(lr_list, epoch_list)
model.fit_generator(
		callbacks=[stepLR]
)