手动更新
第一种是手动更新学习率
optimizer = optim.Adam(model.parameters(), lr=args.lr)
for epoch in range(args.num_epochs):
for param_group in optimizer.param_groups:
param_group['lr'] = args.lr * (0.1 ** (epoch // int(args.num_epochs * 0.8)))
自动更新
第二种是自动更新学习率
导入包
from torch.optim.lr_scheduler import StepLR, MultiStepLR, LambdaLR, ExponentialLR, CosineAnnealingLR, ReduceLROnPlateau
import torch
import torch.nn as nn
LambdaLR
new_lr = k * initial_lr , k是由epoch为自变量算出来的值
initial_lr = 0.1
optimzer_1 = torch.optim.Adam(Net.parameters(),lr = initial_lr)
scheduler_1 = LambdaLR(optimzer_1, lr_lambda=lambda epoch: 1/(epoch + 1)) #y = 1 /(epoch + 1)
print("初始化的学习率:",optimzer_1.defaults['lr'])
for epoch in range(1,11):
optimzer_1.zero_grad()
optimzer_1.step()
print("第%d个epoch的学习率:%f" % (epoch, optimzer_1.param_groups[0]['lr']))
scheduler_1.step()
StepLR
以epoch等间隔的更新学习率,每次更新-----学习率*gmma
initial_lr = 0.1
optimzer_1 = torch.optim.Adam(Net.parameters(),lr = initial_lr)
scheduler_1 = StepLR(optimzer_1, step_size=2, gamma=0.1)
print("初始化的学习率:",optimzer_1.defaults['lr'])
for epoch in range(1,11):
optimzer_1.zero_grad()
optimzer_1.step()
print("第%d个epoch的学习率:%f" % (epoch, optimzer_1.param_groups[0]['lr']))
scheduler_1.step()
MultiStepLR
在指定的epoch上更新学习率
initial_lr = 0.1
optimzer_1 = torch.optim.Adam(Net.parameters(),lr = initial_lr)
scheduler_1 = MultiStepLR(optimzer_1, milestones=[2,6,8], gamma=0.5)
print("初始化的学习率:",optimzer_1.defaults['lr'])
for epoch in range(1,11):
optimzer_1.zero_grad()
optimzer_1.step()
print("第%d个epoch的学习率:%f" % (epoch, optimzer_1.param_groups[0]['lr']))
scheduler_1.step()
CosineAnnealingLR
一种余弦的学习率调整方式, 变化为一个周期
new_lr = eta_min + (initial - eta_min) * (1 + cos(epoch * π/T_max))
import matplotlib.pyplot as plt
initial_lr = 0.1
optimzer_1 = torch.optim.Adam(Net.parameters(),lr = initial_lr)
scheduler_1 = CosineAnnealingLR(optimzer_1, T_max=10)
print("初始化的学习率:",optimzer_1.defaults['lr'])
lr_list = []
for epoch in range(1,101):
optimzer_1.zero_grad()
optimzer_1.step()
print("第%d个epoch的学习率:%f" % (epoch, optimzer_1.param_groups[0]['lr']))
lr_list.append(optimzer_1.param_groups[0]['lr'])
scheduler_1.step()
#画出epoch的变化图
plt.plot(list(range(1,101)),lr_list)
plt.xlabel("epoch")
plt.ylabel("lr")
plt.title("learning rate curve !")
plt.show()