李沐深度学习-丢弃法从零开始and简洁实现

import torch
import torch.nn as nn
import numpy as np
import sys

sys.path.append("路径")
import d2lzh_pytorch as d2l

'''
--------------------------------------下面的dropout函数将以p概率丢弃X中的元素
'''

num_inputs, num_outputs, num_hidden1, num_hidden2 = 784, 10, 256, 256
drop1, drop2 = 0.2, 0.5
num_epochs, lr, batch_size = 5, 100.0, 256
train_loss = torch.nn.CrossEntropyLoss()
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)


class DropSelect:

    def __init__(self, num):
        self.num = num

        if num == 0:  # 丢弃法从零开始
            def dropout(X, p):
                X = X.float()
                assert 0 <= p <= 1  # 对输入的p进行检查，是否在0-1 区间之内，否则停止程序运行
                keep_p = 1 - p  # 存留概率
                if keep_p == 0:
                    return torch.zeros_like(X)  # 存留概率为0，把X中所有元素全部丢弃掉，生成了一个新的张量
                mask = (torch.rand(X.shape) < keep_p).float()  # h'i=δi/(1-p)*hi
                # mask的列代表隐藏层中隐藏单元的个数，mask随机性导致每一行0位置不同，即代表每一个样本它的丢弃隐藏单元的位置也不一样
                # 生成了一个和X同型的tensor，然后做了判断转为变量为true，false的tensor，之后用float函数转化成1,0变量值
                # mask是一个与X同型但变量为0,1的一个稀疏矩阵    return mask * X / keep_p  # X相当于原hi，keep_p相当于1-p，mask相当于εi   返回一个新h'i
                # 即设随机变量δ为0和1的概率分别为p，1-p。使用丢弃法时计算新的隐藏单元h'i=δi/(1-p)*hi
                return mask * X / keep_p

            '''
            ---------------------------------------------------------定义模型参数 只有tensor才有梯度要求
            '''
            w1 = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_hidden1)), dtype=torch.float,
                              requires_grad=True)
            b1 = torch.zeros((1, num_hidden1), requires_grad=True)
            w2 = torch.tensor(np.random.normal(0, 0.01, (num_hidden1, num_hidden2)), dtype=torch.float,
                              requires_grad=True)
            b2 = torch.zeros((1, num_hidden2), requires_grad=True)
            w3 = torch.tensor(np.random.normal(0, 0.01, (num_hidden2, num_outputs)), dtype=torch.float,
                              requires_grad=True)
            b3 = torch.zeros((1, num_outputs), requires_grad=True)

            params = [w1, b1, w2, b2, w3, b3]

            '''
            --------------------------------------------------------定义模型，丢弃法编程时应用在激活函数的输出上
            分别设置每个层的丢弃概率，一般把靠近输入层的丢弃概率设置小一点，只在训练模式下使用丢弃法
            '''

            # 此处，X是mnist传进来的数据集，要对其进行整型
            def net(X, is_training=True):
                X = X.view(-1, num_inputs)
                H1 = (torch.matmul(X, w1) + b1).relu()
                if is_training:
                    H1 = dropout(H1, drop1)  # 在第一层全连接后添加丢弃层  H1是一个NxNum_inputs大小的格式，每个元素都是一次模型计算后的值
                H2 = (torch.matmul(H1, w2) + b2).relu()
                if is_training:
                    H2 = dropout(H2, drop2)  # 在第二层全连接层后添加丢弃层
                return torch.matmul(H2, w3) + b3

            '''
            -------------------------------------------------------训练和测试模型
            '''

            d2l.train_ch3(net, train_iter, test_iter, train_loss, num_epochs, batch_size, params, lr)

        else:
            '''
            ----------------------------------------------------丢弃法简洁实现
            只需要在全连接层后添加Dropout层并指定概率，
            训练模型时，dropout将以指定概率随机丢弃上一层输出元素
            在测试模型时（即model.eval()后）,Dropout层不起作用
            '''

            net = nn.Sequential(
                d2l.FlattenLayer(),
                nn.Linear(num_inputs, num_hidden1),
                nn.ReLU(),
                nn.Dropout(drop1),  # 丢弃
                nn.Linear(num_hidden1, num_hidden2),
                nn.ReLU(),
                nn.Dropout(drop2),  # 丢弃
                nn.Linear(num_hidden2, num_outputs)
            )
            for param in net.parameters():
                nn.init.normal_(param, mean=0, std=0.01)
            optimizer = torch.optim.SGD(net.parameters(), lr=0.5, weight_decay=0)  # 衰退设置为0

            def train():
                for epoch in range(num_epochs):
                    train_l, train_acc, test_acc, n, m = 0.0, 0.0, 0.0, 0, 0
                    for X, y in train_iter:
                        l = train_loss(net(X), y).sum()
                        optimizer.zero_grad()
                        l.backward()
                        optimizer.step()
                        n += y.shape[0]
                        m += 1
                        train_l += l.item()
                        train_acc += (net(X).argmax(dim=1) == y).float().sum().item()
                    test_acc += d2l.evaluate_accuracy(test_iter, net)
                    print(f'epoch %d, loss %.4f, train_acc %.3f, test_acc %.3f'
                          % (epoch, train_l / n, train_acc / n, test_acc))

            train()


DropSelect(1)