import torch
import torch.nn as nn
import numpy as np
import sys
sys.path.append("路径")
import d2lzh_pytorch as d2l
'''
--------------------------------------下面的dropout函数将以p概率丢弃X中的元素
'''
num_inputs, num_outputs, num_hidden1, num_hidden2 = 784, 10, 256, 256
drop1, drop2 = 0.2, 0.5
num_epochs, lr, batch_size = 5, 100.0, 256
train_loss = torch.nn.CrossEntropyLoss()
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
class DropSelect:
def __init__(self, num):
self.num = num
if num == 0: # 丢弃法从零开始
def dropout(X, p):
X = X.float()
assert 0 <= p <= 1 # 对输入的p进行检查,是否在0-1 区间之内,否则停止程序运行
keep_p = 1 - p # 存留概率
if keep_p == 0:
return torch.zeros_like(X) # 存留概率为0,把X中所有元素全部丢弃掉,生成了一个新的张量
mask = (torch.rand(X.shape) < keep_p).float() # h'i=δi/(1-p)*hi
# mask的列代表隐藏层中隐藏单元的个数,mask随机性导致每一行0位置不同,即代表每一个样本它的丢弃隐藏单元的位置也不一样
# 生成了一个和X同型的tensor,然后做了判断转为变量为true,false的tensor,之后用float函数转化成1,0变量值
# mask是一个与X同型但变量为0,1的一个稀疏矩阵 return mask * X / keep_p # X相当于原hi,keep_p相当于1-p,mask相当于εi 返回一个新h'i
# 即设随机变量δ为0和1的概率分别为p,1-p。使用丢弃法时计算新的隐藏单元h'i=δi/(1-p)*hi
return mask * X / keep_p
'''
---------------------------------------------------------定义模型参数 只有tensor才有梯度要求
'''
w1 = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_hidden1)), dtype=torch.float,
requires_grad=True)
b1 = torch.zeros((1, num_hidden1), requires_grad=True)
w2 = torch.tensor(np.random.normal(0, 0.01, (num_hidden1, num_hidden2)), dtype=torch.float,
requires_grad=True)
b2 = torch.zeros((1, num_hidden2), requires_grad=True)
w3 = torch.tensor(np.random.normal(0, 0.01, (num_hidden2, num_outputs)), dtype=torch.float,
requires_grad=True)
b3 = torch.zeros((1, num_outputs), requires_grad=True)
params = [w1, b1, w2, b2, w3, b3]
'''
--------------------------------------------------------定义模型,丢弃法编程时应用在激活函数的输出上
分别设置每个层的丢弃概率,一般把靠近输入层的丢弃概率设置小一点,只在训练模式下使用丢弃法
'''
# 此处,X是mnist传进来的数据集,要对其进行整型
def net(X, is_training=True):
X = X.view(-1, num_inputs)
H1 = (torch.matmul(X, w1) + b1).relu()
if is_training:
H1 = dropout(H1, drop1) # 在第一层全连接后添加丢弃层 H1是一个NxNum_inputs大小的格式,每个元素都是一次模型计算后的值
H2 = (torch.matmul(H1, w2) + b2).relu()
if is_training:
H2 = dropout(H2, drop2) # 在第二层全连接层后添加丢弃层
return torch.matmul(H2, w3) + b3
'''
-------------------------------------------------------训练和测试模型
'''
d2l.train_ch3(net, train_iter, test_iter, train_loss, num_epochs, batch_size, params, lr)
else:
'''
----------------------------------------------------丢弃法简洁实现
只需要在全连接层后添加Dropout层并指定概率,
训练模型时,dropout将以指定概率随机丢弃上一层输出元素
在测试模型时(即model.eval()后),Dropout层不起作用
'''
net = nn.Sequential(
d2l.FlattenLayer(),
nn.Linear(num_inputs, num_hidden1),
nn.ReLU(),
nn.Dropout(drop1), # 丢弃
nn.Linear(num_hidden1, num_hidden2),
nn.ReLU(),
nn.Dropout(drop2), # 丢弃
nn.Linear(num_hidden2, num_outputs)
)
for param in net.parameters():
nn.init.normal_(param, mean=0, std=0.01)
optimizer = torch.optim.SGD(net.parameters(), lr=0.5, weight_decay=0) # 衰退设置为0
def train():
for epoch in range(num_epochs):
train_l, train_acc, test_acc, n, m = 0.0, 0.0, 0.0, 0, 0
for X, y in train_iter:
l = train_loss(net(X), y).sum()
optimizer.zero_grad()
l.backward()
optimizer.step()
n += y.shape[0]
m += 1
train_l += l.item()
train_acc += (net(X).argmax(dim=1) == y).float().sum().item()
test_acc += d2l.evaluate_accuracy(test_iter, net)
print(f'epoch %d, loss %.4f, train_acc %.3f, test_acc %.3f'
% (epoch, train_l / n, train_acc / n, test_acc))
train()
DropSelect(1)
李沐深度学习-丢弃法从零开始and简洁实现
最新推荐文章于 2024-08-21 21:39:04 发布
![](https://img-home.csdnimg.cn/images/20240711042549.png)