LearningRateScheduler class
tf.keras.callbacks.LearningRateScheduler(schedule, verbose=0)
参数:
- schedule : 一个以epoch index和当前lr rate 为参数的函数。
- verbose : 是否显示 更新信息。 0 1
# -*- coding: utf-8 -*-
# author:
# time : 2020/12/21 21:31
import tensorflow as tf
from keras.callbacks import LearningRateScheduler
import numpy as np
def scheduler(epoch, lr):
if epoch < 10:
return lr
else:
return lr * tf.math.exp(-0.1)
model = tf.keras.models.Sequential([tf.keras.layers.Dense(units=10)])
model.compile(optimizer=tf.keras.optimizers.SGD(), loss='mse')
t = round(model.optimizer.lr.numpy(), 5)
print(t)
# 0.01
callback = LearningRateScheduler(scheduler)
# history = model.fit(np.arange(100).reshape(5, 20), np.zeros(5), epochs=15, callbacks=[callback], verbose=0)
# 训练15次后,我们查看下学习率的改变情况。
history = model.fit(np.arange(100).reshape(5, 20), np.zeros(5),
epochs=15, callbacks=[callback], verbose=0)
print(round(model.optimizer.lr.numpy(), 5))
# 0.00607
更加复杂的函数:
def exponential_lr(epoch,
start_lr = 0.00001, min_lr = 0.00001, max_lr = 0.00005,
rampup_epochs = 5, sustain_epochs = 0,
exp_decay = 0.8):
'''
:param epoch: 当前epoch
:param start_lr: 开始的学习率。
:param min_lr: 最小的学习率。
:param max_lr: 最大学习率。
:param rampup_epochs: 最大学习率对应的epoch
:param sustain_epochs: 最大学习率持续epoch数量。
:param exp_decay: # 衰减因子。
:return: 这个函数与当前学习率无关,只与最大最小的学习率,以及当前epoch有关。
'''
def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):
# linear increase from start to rampup_epochs
if epoch < rampup_epochs:
lr = ((max_lr - start_lr) /
rampup_epochs * epoch + start_lr) #rampup_epochs 这部分是线性增加的,直到增加到最大的max_lr
# constant max_lr during sustain_epochs
elif epoch < rampup_epochs + sustain_epochs:
lr = max_lr # 最大的学习率持续的长度。
# exponential decay towards min_lr
else:
lr = ((max_lr - min_lr) *
exp_decay**(epoch - rampup_epochs - sustain_epochs) +
min_lr) # 0.8**(超出epoch数量)乘以最大最小的lr间隔,加上最小的lr
return lr
return lr(epoch,
start_lr,
min_lr,
max_lr,
rampup_epochs,
sustain_epochs,
exp_decay)
实验结果:
# -*- coding: utf-8 -*-
# author: cuihu
# time : 2020/12/21 22:04
# 更加复杂的情况。
def exponential_lr(epoch,
start_lr = 0.00001, min_lr = 0.00001, max_lr = 0.00005,
rampup_epochs = 5, sustain_epochs = 0,
exp_decay = 0.8):
'''
:param epoch: 当前epoch
:param start_lr: 开始的学习率。
:param min_lr: 最小的学习率。
:param max_lr: 最大学习率。
:param rampup_epochs: 最大学习率对应的epoch
:param sustain_epochs: 最大学习率持续epoch数量。
:param exp_decay: # 衰减因子。
:return: 这个函数与当前学习率无关,只与最大最小的学习率,以及当前epoch有关。
'''
def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):
# linear increase from start to rampup_epochs
if epoch < rampup_epochs:
lr = ((max_lr - start_lr) /
rampup_epochs * epoch + start_lr) #rampup_epochs 这部分是线性增加的,直到增加到最大的max_lr
# constant max_lr during sustain_epochs
elif epoch < rampup_epochs + sustain_epochs:
lr = max_lr # 最大的学习率持续的长度。
# exponential decay towards min_lr
else:
lr = ((max_lr - min_lr) *
exp_decay**(epoch - rampup_epochs - sustain_epochs) +
min_lr) # 0.8**(超出epoch数量)乘以最大最小的lr间隔,加上最小的lr
return lr
return lr(epoch,
start_lr,
min_lr,
max_lr,
rampup_epochs,
sustain_epochs,
exp_decay)
import tensorflow as tf
from keras.callbacks import LearningRateScheduler
import numpy as np
model = tf.keras.models.Sequential([tf.keras.layers.Dense(units=10)])
model.compile(optimizer=tf.keras.optimizers.SGD(), loss='mse')
t = round(model.optimizer.lr.numpy(), 5)
print(t)
callback = LearningRateScheduler(exponential_lr)
history = model.fit(np.arange(100).reshape(5, 20), np.zeros(5),
epochs=15, callbacks=[callback], verbose=0)
print(round(model.optimizer.lr.numpy(), 5))
结果:
0.01
2e-05
我们将学习率与epoch的关系画出来:
import matplotlib.pyplot as plt
rng = [i for i in range(20)]
y = [exponential_lr(x,sustain_epochs=3) for x in rng]
plt.plot(rng, y)
plt.show()
可以看出,前面为线性增长,中间保持不变,最后指数形式下降。