pytorch之学习率变化策略之MultiplicativeLR

MultiplicativeLR

torch.optim.lr_scheduler.MultiplicativeLR(optimizer, lr_lambda, last_epoch=-1, verbose=False)
设置学习率为上一次的学习率乘以给定lr_lambda函数的值
new_lr = lr_lambda(self.last_epoch) * last_lr
last_lr 最开始为base_lr

• optimizer：优化器
• lr_lambda：函数或者函数列表
• last_epoch：默认为-1，学习率更新计数；注意断点训练时last_epoch不为-1

例子

# -*- coding：utf-8 -*-
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import torch
from torch.optim.lr_scheduler import MultiplicativeLR
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
import random
from torch.nn import CrossEntropyLoss
import matplotlib.pyplot as plt

"""上一次的学习率乘lambda1的函数值得到新的学习率"""


# 定义模型
class Net(nn.Module):
    def __init__(self, n_feature, n_hidden, n_out):
        super(Net, self).__init__()
        self.hidden = nn.Linear(n_feature, n_hidden)
        self.out = nn.Linear(n_hidden, n_out)
        self.init_weights()

    def init_weights(self):
        initrange = 0.5
        self.hidden.weight.data.uniform_(-initrange, initrange)
        self.hidden.bias.data.zero_()
        self.out.weight.data.uniform_(-initrange, initrange)
        self.out.bias.data.zero_()

    def forward(self, x, y=None):
        x = self.hidden(x)
        x = torch.sigmoid(x)
        x = self.out(x)
        out = F.log_softmax(x, dim=1)
        loss = None
        if y is not None:
            loss_fct = CrossEntropyLoss()
            loss = loss_fct(out, y)
        return out, loss


# 构造数据
data_x = [torch.randn(32, 50)] * 16
data_y = [[1 if random.random() > 0.5 else 0 for j in range(32)]] * 16

# 模型
net = Net(n_feature=50, n_hidden=10, n_out=2)

# 优化器
optimizer = optim.Adam(net.parameters(), lr=1e-3)

# 学习率变化策略
lambda1 = lambda epoch: 0.95
# epoch仅仅是函数的自变量,代表scheduler.step()次数,与训练模型中的epoch不同

# lambda2 = lambda epoch: 0.95 ** epoch
# 如果有多组参数，可以设置不同的组使用不同的学习率变化策略

scheduler = MultiplicativeLR(optimizer, lr_lambda=lambda1, last_epoch=-1)
print(scheduler.base_lrs[0])
print(scheduler.get_lr()[0])
print(scheduler.last_epoch)
print("=====================================================================")

"""上一次的学习率乘lambda1的函数值得到新的学习率"""

# 画图
x_plot = []
y_plot = []
last_lr = optimizer.param_groups[0]["lr"]

for epoch in range(10):
    for step, batch in enumerate(zip(data_x, data_y)):
        x, y = batch
        y = torch.tensor(y)
        out, loss = net(x, y)
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
        scheduler.step()
        x_plot.append(scheduler.last_epoch)
        y_plot.append(scheduler.get_lr()[0])
        print(lambda1(scheduler.last_epoch))
        print(last_lr)
        print(optimizer.param_groups[0]["lr"])
        assert lambda1(scheduler.last_epoch) * last_lr == optimizer.param_groups[0]["lr"], "error"
        last_lr = optimizer.param_groups[0]["lr"]

plt.plot(x_plot, y_plot, )
plt.savefig('./MultiplicativeLR.jpg')