【lzy学习笔记-dive into deep learning】4.5 权重衰减 原理与代码实现

# 从0开始的实现

import torch
from torch import nn
import commfuncs

# 为了使过拟合的效果更加明显，可以将问题的维数增加到d=200，并使⽤⼀个只包含20个样本的小训练集
n_train, n_test, num_inputs, batch_size = 20, 100, 200, 5
true_w, true_b = torch.ones((num_inputs, 1)) * 0.01, 0.05
train_data = commfuncs.synthetic_data(true_w, true_b, n_train)
train_iter = commfuncs.load_array(train_data, batch_size)
test_data = commfuncs.synthetic_data(true_w, true_b, n_test)
test_iter = commfuncs.load_array(test_data, batch_size, is_train=False)

def init_params():
    w = torch.normal(0, 1, size=(num_inputs, 1), requires_grad=True)
    b = torch.zeros(1, requires_grad=True)
    return [w, b]

def l2_penalty(w):
    return torch.sum(w.pow(2)) / 2

def train(lambd):
    w, b = init_params()
    net, loss = lambda X: commfuncs.linreg(X, w, b), commfuncs.squared_loss
    num_epochs, lr = 100, 0.003
    for epoch in range(num_epochs):
        for X, y in train_iter:
            l = loss(net(X), y) + lambd * l2_penalty(w)
            l.sum().backward()
            commfuncs.sgd([w, b], lr, batch_size)
        print(f'epoch {epoch + 1}, train loss {commfuncs.evaluate_loss(net, train_iter, loss): f}, '
              f'test loss {commfuncs.evaluate_loss(net, test_iter, loss): f}')
    print('L2 norm of w:', torch.norm(w).item())

train(lambd=0) # 未正则化
# L2 norm of w: 11.72992992401123
# epoch 100, train loss  0.000056, test loss  68.219870

train(lambd=3) # 正则化
# L2 norm of w: 0.3580736517906189
# epoch 100, train loss  0.000534, test loss  0.061504

由于权重衰减在神经⽹络优化中很常⽤，深度学习框架为了便于我们使⽤权重衰减，将权重衰减集成到优化算法中，以便与任何损失函数结合使⽤。此外，这种集成还有计算上的好处，允许在不增加任何额外的计算开销的情况下向算法中添加权重衰减。
由于更新的权重衰减部分仅依赖于每个参数的当前值，因此优化器必须⾄少接触每个参数⼀次。
用框架实现运⾏得更快，更容易实现。对于更复杂的问题，这⼀好处将变得更加明显。

# 深度学习框架API实现
import torch
from torch import nn
import commfuncs

n_train, n_test, num_inputs, batch_size = 20, 100, 200, 5
true_w, true_b = torch.ones((num_inputs, 1)) * 0.01, 0.05
train_data = commfuncs.synthetic_data(true_w, true_b, n_train)
train_iter = commfuncs.load_array(train_data, batch_size)
test_data = commfuncs.synthetic_data(true_w, true_b, n_test)
test_iter = commfuncs.load_array(test_data, batch_size, is_train=False)

def train_concise(wd):
    net = nn.Sequential(nn.Linear(num_inputs, 1))
    for param in net.parameters():
        param.data.normal_()
    loss = nn.MSELoss(reduction='none') # 跳转看MSELoss就明白了
    num_epochs, lr = 100, 0.003
    # 在实例化优化器时直接通过weight_decay指定weight decay超参数。
    # 默认情况下，PyTorch同时衰减权重和偏移。
    # 这只为权重设置了weight_decay，所以偏置参数b不会衰减。
    trainer = torch.optim.SGD([{"params": net[0].weight, 'weight_decay':wd}, {"params": net[0].bias}], lr=lr)
    for epoch in range(num_epochs):
        for X, y in train_iter:
            trainer.zero_grad()
            l = loss(net(X), y)
            l.mean().backward()
            trainer.step()
        print(f'epoch {epoch + 1}, train loss {commfuncs.evaluate_loss(net, train_iter, loss): f}, '
              f'test loss {commfuncs.evaluate_loss(net, test_iter, loss): f}')
    print('L2 norm of w:', net[0].weight.norm().item())


train_concise(0)
# epoch 100, train loss  0.000000, test loss  140.556103
# L2 norm of w: 12.363112449645996

train_concise(3)
# epoch 100, train loss  0.000403, test loss  0.152332
# L2 norm of w: 0.3822719156742096