import torch
import torch.nn as nn
import numpy as np
import sys
sys.path.append("路径")
import d2lzh_pytorch as d2l
'''
该实验为了验证权重衰退的作用,特地设置样本数小于权重数量
样本特征维度=p x1,x2....xp
模型: y=0.05+Σ(i=1,p) 0.01Xi +ε 噪声服从均值0,标准差0.01 正态分布
为了观察过拟合,因该设置训练数据集<模型复杂度(模型参数/特征数)
'''
n_train, n_test, num_inputs = 20, 100, 200
true_w, true_b = torch.ones((num_inputs, 1)) * 0.01, 0.05 # 不是分类问题,所以w只有一列,行数=特征数即可
features = torch.randn((n_train + n_test, num_inputs))
labels = torch.matmul(features, true_w) + true_b
labels += torch.tensor(np.random.normal(0, 0.01, labels.size()), dtype=torch.float)
train_features, test_features = features[:n_train, :], features[n_train:, :]
train_labels, test_labels = labels[:n_train], labels[n_train:]
x = torch.tensor([[1, 2, 3], [1, 2, 3]])
'''
从零开始实现权重衰减:通过在目标函数后添加L2范数惩罚项来实现权重衰减
---------------------------------------------------------------------------初始化模型参数
'''
# 定义随机初始化模型参数的函数
def init_params():
w = torch.randn((num_inputs, 1), requires_grad=True)
b = torch.zeros(1, requires_grad=True)
return [w, b]
'''
-----------------------------------------------------------------------定义范数惩罚项
'''
def l2_penalty(w):
return (w ** 2).sum() / 2 # 为什么要进行sum()操作?
'''
----------------------------------------------------------定义训练和测试
'''
# 定义在训练集和测试集上分别训练和测试模型
batch_size, num_epochs, lr = 1, 100, 0.003
net, loss = d2l.linreg, d2l.square_loss
dataset = torch.utils.data.TensorDataset(train_features, train_labels)
train_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True)
def fit_and_plot(lambd, name):
train_l, test_l = [], []
w, b = init_params()
for epoch in range(num_epochs):
for X, y in train_iter:
y_hat = net(X, w, b) # net计算预测值
# 添加L2惩罚项
l = loss(y_hat, y) + lambd * l2_penalty(w) # 计算预测值和实际值之间的损失
l = l.sum()
if w.grad is not None: # 对参数的梯度清零
w.grad.data.zero_()
b.grad.data.zero_()
l.backward() # 这一步计算了参数的梯度,然后将梯度存储在参数中
d2l.sgd([w, b], lr, batch_size) # 这一步对参数中存储的梯度进行了计算,实现梯度下降
train_l.append((loss(net(train_features, w, b), train_labels)).mean().item())
test_l.append(loss(net(test_features, w, b), test_labels).mean().item())
d2l.semilogy(range(1, num_epochs + 1), train_l, 'epoch', 'loss', name,
range(1, num_epochs + 1), test_l, ['train_l', 'test_l'])
print(f'L2 norm of w:', w.norm().item())
'''
-----------------------------------------------------无权重衰退参与实验
'''
fit_and_plot(0, "无L2范数惩罚项过拟合观察")
# 权重衰退起作用就是使得权重参数w接近于0,减小了模型复杂度
'''
----------------------------------------------------权重衰退参与实验
'''
fit_and_plot(10, "L2范数惩罚项参与过拟合观察")
李沐深度学习-权重衰退从零开始实验
于 2024-01-21 22:39:16 首次发布