#num_train_optimization_steps = int(len(train_data) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs
class WarmupLinearSchedule(_LRSchedule):
"""
Linearly increases learning rate from 0 to 1 over `warmup` fraction of training steps.
Linearly decreases learning rate from 1. to 0. over remaining `1 - warmup` steps.
"""
warn_t_total = True
def get_lr_(self, progress):
# progerss从0开始
if progress < self.warmup:
return progress / self.warmup
return max((progress - 1.) / (self.warmup - 1.), 0.)
设self.warmup=0.3,表示前30%的epoch学习率逐步上升,后70%逐步下降。
progress=0.1时, progress / self.warmup=1/3
progress=0.2时, progress / self.warmup=2/3
progress=0.3时, progress / self.warmup=3/3
progress=0.4时, (progress - 1.) / (self.warmup - 1.)=6/7
progress=0.5时, (progress - 1.) / (self.warmup - 1.)=5/7
progress=0.6时, (progress - 1.) / (self.warmup - 1.)=4/7
progress=0.7时, (progress - 1.) / (self.warmup - 1.)=3/7
progress=0.8时, (progress - 1.) / (self.warmup - 1.)=2/7
progress=0.9时, (progress - 1.) / (self.warmup - 1.)=1/7
progress / self.warmup
和(progress - 1.) / (self.warmup - 1.)
相当于控制学习率增长的一个系数。
# 在BertAdam中,group['schedule'].get_lr(state['step'])就会调用WarmupLinearSchedule函数,前提是使用线性的Schedule
lr_scheduled = group['lr']
lr_scheduled *= group['schedule'].get_lr(state['step'])