动手学pytorch DNN

tool.py

#coding=utf-8
import collections
import math
import os
import random
import sys
import tarfile
import time
import zipfile
# from tqdm import tqdm

# from IPython import display
import numpy as np
from matplotlib import pyplot as plt

import torch
from torch import nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
# import torchtext
# import torchtext.vocab as Vocab
import sklearn

"""dataset"""
# 批量读取数据
def data_iter(batch_size, features, labels):
    num_examples = len(features)
    indices = list(range(num_examples))
    random.shuffle(indices)  # 样本的读取顺序是随机的
    for i in range(0, num_examples, batch_size):
        j = torch.LongTensor(indices[i: min(i + batch_size, num_examples)]) # 最后一次可能不足一个batch
        yield  features.index_select(0, j), labels.index_select(0, j)


""" Fashion-MNIST 数据集 p42"""
# 获取 mnist 数据集
def load_data_fashion_mnist(batch_size, resize=None, root='~/Datasets/FashionMNIST'):
    """Download the fashion mnist dataset and then load into memory."""
    trans = []
    if resize:
        trans.append(torchvision.transforms.Resize(size=resize))
    trans.append(torchvision.transforms.ToTensor())

    transform = torchvision.transforms.Compose(trans)
    mnist_train = torchvision.datasets.FashionMNIST(root=root, train=True, download=True, transform=transform)
    mnist_test = torchvision.datasets.FashionMNIST(root=root, train=False, download=True, transform=transform)
    if sys.platform.startswith('win'):
        num_workers = 0  # 0表示不用额外的进程来加速读取数据
    else:
        num_workers = 4
    train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=num_workers)
    test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False, num_workers=num_workers)
    return train_iter, test_iter

"""
train_iter, test_iter = datasets.load_data_fashion_mnist(batch_size=50)
for X,y in train_iter:
        print(X.shape, y.shape)         >>>torch.Size([50, 1, 28, 28]) torch.Size([50])
        print(X.view((-1, 784)).shape)  >>>torch.Size([50, 784])
        break

"""

def get_fashion_mnist_labels(labels):
    text_labels = ['t-shirt', 'trouser', 'pullover', 'dress', 'coat',
                   'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
    return [text_labels[int(i)] for i in labels]

def show_fashion_mnist(images, labels):
    use_svg_display()
    # 这里的_表示我们忽略（不使用）的变量
    _, figs = plt.subplots(1, len(images), figsize=(12, 12))
    for f, img, lbl in zip(figs, images, labels):
        f.imshow(img.view((28, 28)).numpy())
        f.set_title(lbl)
        f.axes.get_xaxis().set_visible(False)
        f.axes.get_yaxis().set_visible(False)
    # plt.show()

def set_figsize(figsize=(3.5, 2.5)):
    use_svg_display()
    # 设置图的尺寸
    plt.rcParams['figure.figsize'] = figsize

def use_svg_display():
    """Use svg format to display plot in jupyter"""
    display.set_matplotlib_formats('svg')




# 线性回归
def linreg(X, w, b):
    return torch.mm(X, w) + b


# 均方误差
def squared_loss(y_hat, y):
    """
    y_hat = torch.tensor([0.1, 2.2, 2.6])
    y     = torch.tensor([0.0, 2., 3.])
    tool.squared_loss(y_hat, y)
    >> tensor([0.0050, 0.0200, 0.0800])  # 除以2
    """
    # 注意这里返回的是向量, 另外, pytorch里的MSELoss并没有除以 2
    return ((y_hat - y.view(y_hat.size())) ** 2) / 2

# softmax 对应于书P40上
def softmax(X):
    """
    :param X: tensor m*n
    :return:  tensor m*n 且任一行之和为1

    x = torch.tensor([[1.2, 2.2, 3.2],
                      [3.3, 4.4, 5.5]])
    print(tool.softmax(x))
    >> tensor([[0.0900, 0.2447, 0.6652],
               [0.0768, 0.2306, 0.6927]])

    """
    X_exp = X.exp()
    partition = X_exp.sum(dim=1, keepdim=True)
    return X_exp / partition  # 这里应用了广播机制

# soft_net 对应于书P40下
def soft_net(X, W, b):
    """
    :param X: 输入变量 tensor m*num_inputs
    :return:  输出 tensor m*num_outputs
    """
    return softmax(torch.mm(X, W) + b)


# 交叉熵 上式对应于P41的小公式
def cross_entropy(y_hat, y):
    """
    :param y_hat:
    :param y:
    :return:
    该函数等同于 nn.CrossEntropyLoss()

    y_hat = torch.tensor([[0.1, 0.3, 0.6],
                          [0.3, 0.2, 0.5]])
    y = torch.LongTensor([0, 2])
    loss2 = torch.nn.CrossEntropyLoss()
    output1 = loss2(y_hat, y)
    >> tensor(1.1466)
    output2 = tool.cross_entropy(y_hat, y)
    >> tensor(1.4979)
    print(-(math.log(0.1) + math.log(0.5)) / 2)  # 对应的数学计算过程
    >> 1.4978661367769954
    """
    return - torch.log(y_hat.gather(1, y.view(-1, 1))).mean()

"""
torch.gather(input, dim, index, out=None) → Tensor
    Gathers values along an axis specified by dim.
    
print(y_hat.gather(1, y.view(-1, 1)))  >>>tensor([[0.1000],[0.5000]])
"""


# 随机梯度下降
def sgd(params, lr, batch_size):
    # 为了和原书保持一致，这里除以了batch_size，但是应该是不用除的，因为一般用PyTorch计算loss时就默认已经
    # 沿batch维求了平均了。
    for param_u in params:
        for param in param_u:
            param.data -= lr * param.grad / batch_size # 注意这里更改param时用的param.data

# softmax准确度 p47
def accuracy_for_softmax(y_hat, y):
    """
    给定类别的预测概率分布y_hat, 判断与真是类别y是否一致
    :param y_hat: 预测概率 m*n
    :param y:     真实类别 m*1
    :return:
    y_hat = torch.tensor([[0.1, 0.3, 0.6],
                          [0.3, 0.2, 0.5]])
    y = torch.LongTensor([0, 2])
    >> 0.5
    """
    return (y_hat.argmax(dim=1) == y).float().mean().item()


def evaluate_classifier(y_hat, y):
    """
    y_hat = torch.tensor([[0.1, 0.3, 0.6],
                          [0.3, 0.2, 0.5]])
    y = torch.LongTensor([0, 2])
    >> 1.0
    """
    return (y_hat.argmax(dim=1) == y).float().sum().item()


def evaluate_regression(y_hat, y):
    """
    y_hat = torch.tensor([0.1, 2.2, 2.6])
    y     = torch.tensor([0.0, 2., 3.])
    tool.evaluate_regression(y_hat, y)
    >> 0.10500004887580872 (实际上是0.10500, 在item()之后出现一点误差)
    """
    return squared_loss(y_hat, y).float().sum().item()


# 评估softmax模型net在数据集data_iter上的准确率 p47
def evaluate_accuracy_for_softmax(data_iter, net):
    acc_sum, n = 0.0, 0
    for X, y in data_iter:
        # acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()
        acc_sum += evaluate_classifier(net(X), y)
        n += y.shape[0]
    return acc_sum / n

def evaluate_accuracy_for_regression(data_iter, net):
    acc_sum, n = 0.0, 0.1
    for X, y in data_iter:
        # acc_sum += squared_loss(net(X), y).float().sum().item()
        acc_sum += evaluate_regression(net(X), y)
        n += y.shape[0]
    return acc_sum / n


# 训练函数 p48
def train_ch3(net, train_iter, test_iter, loss,  batch_size, need_softmax=False,
              num_epochs=50, params=None, lr=0.1, optimizer=None):
    """
    X.view((-1, num_inputs)
    :param net:            list     例如：[relu, tanh, 'None']
    :param train_iter:     dataloader
    :param test_iter:      datalodaer
    :param loss:                   例如： squared_loss, cross_entropy
    :param batch_size:   批量
    :param num_epochs:   轮次
    :param params:        list    例如: [ [w1, b1], [w2, b2] ]
    :param lr:
    :param optimizer:
    :return:

    使用：
    train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, batch_size, [W, b], lr)
    """
    for epoch in range(num_epochs):
        train_l_sum, train_acc_sum, n = 0.0, 0.0, 0

        for X, y in train_iter:
            input = X.view(-1, 784)
            for i, param_u in enumerate(params):    # 多层实现
                w, b = param_u
                nn = net[i]
                if nn == 'None':
                    output = linreg(input, w, b)
                else:
                    output = nn(linreg(input, w, b))
                input = output
            y_hat = input

            if need_softmax == True:
                y_hat = softmax(y_hat)

            l = loss(y_hat, y).sum()

            # 梯度清零
            if optimizer is not None:
                optimizer.zero_grad()
            elif params is not None and params[0][0].grad is not None:
                for param_u in params:
                    for param in param_u:
                        param.grad.data.zero_()

            l.backward()
            if optimizer is None:
                sgd(params, lr, batch_size)
            else:
                optimizer.step()  # “softmax回归的简洁实现”一节将用到

            train_l_sum += l.item()
            train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()
            n += y.shape[0]
        # test_acc = evaluate_accuracy(test_iter, net)
        # print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
        #       % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))
        print('epoch %d, loss %.4f, train acc %.3f'
        % (epoch + 1, train_l_sum / n, train_acc_sum / n))

model.py (scratch)

#coding=utf-8
# %matplotlib inline
import torch
import torch.nn as nn
import torch.nn.functional as F
# from IPython import display
# from matplotlib import pyplot as plt
import numpy as np
import random
import tool

print(torch.__version__)
torch.set_default_tensor_type('torch.FloatTensor')

class MyMLP:
    def __init__(self, node_list, acti_list, classifier=True):
        """
        :param node_list: list  节点数的列表
        """
        self.params = []
        self.acti_list = acti_list
        self.num_layer = len(node_list)-1
        self.classifier = classifier
        self.inputs_num = node_list[0]
        self.outputs_num = node_list[-1]

        num_inputs = node_list[0]
        for n in node_list[1:]:
            num_outputs = n
            w = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_outputs)), dtype=torch.float)
            b = torch.zeros(num_outputs)
            w.requires_grad_()
            b.requires_grad_()
            self.params.append([w, b])
            num_inputs = num_outputs
        # self.w = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_outputs)), dtype=torch.float)
        # self.b = torch.zeros(num_outputs)
        # self.w.requires_grad_()
        # self.b.requires_grad_()


    # def fit(self, train_iter, test_iter, lr = 0.01, num_epochs=5, batch_size = 50,  loss = tool.cross_entropy, optimizer=None):
    #
    #     params = [[self.w, self.b]]
    #     net = ['None']
    #     need_softmax = True
    #     # train_iter, test_iter = tool.load_data_fashion_mnist(batch_size)
    #     tool.train_ch3(net, train_iter, test_iter, loss, batch_size, need_softmax, num_epochs, params, lr, optimizer)


    def fit(self, train_iter, test_iter, lr = 0.01, num_epochs=10, batch_size = 50, optimizer=None):
        if self.classifier == True:
            loss = tool.cross_entropy
            evaluate = tool.evaluate_classifier
            evaluate_test = tool.evaluate_accuracy_for_softmax

        else:
            loss = tool.squared_loss
            evaluate = tool.evaluate_regression
            evaluate_test = tool.evaluate_accuracy_for_regression

        for epoch in range(num_epochs):
            train_l_sum, train_acc_sum, n = 0.0, 0.0, 0.1

            for X, y in train_iter:  # 批次训练
                # input = X.view(-1, 784)
                input = X.view(-1, self.inputs_num)
                """前向传播过程"""
                for i, param_u in enumerate(self.params):  # MLP多层实现
                    w, b = param_u
                    activation = self.acti_list[i]

                    if activation == 'relu':
                        output = F.relu(tool.linreg(input, w, b))
                    elif activation == 'tanh':
                        output = F.tanh(tool.linreg(input, w, b))
                    elif activation == 'selu':
                        output = F.selu(tool.linreg(input, w, b))
                    elif activation == 'sigmoid':
                        output = F.sigmoid(tool.linreg(input, w, b))
                    else:
                        output = tool.linreg(input, w, b)
                    input = output
                y_hat = input

                if self.classifier == True:   # 输出层是否需要softmax
                    y_hat = tool.softmax(y_hat)

                """反向传播过程"""
                l = loss(y_hat, y).sum()

                # 梯度清零
                if optimizer is not None:
                    optimizer.zero_grad()
                elif self.params is not None and self.params[0][0].grad is not None:
                    for param_u in self.params:
                        for param in param_u:
                            param.grad.data.zero_()
                # 反向求导
                l.backward()
                if optimizer is None:
                    tool.sgd(self.params, lr, batch_size)  # sgd
                else:
                    optimizer.step()  # “softmax回归的简洁实现”一节将用到


                """评估"""
                train_l_sum += l.item()
                train_acc_sum += evaluate(y_hat, y)
                n += y.shape[0]
            test_acc = evaluate_test(test_iter, self.predict)
            print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
                  % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))
            # print('epoch %d, loss %.4f, train acc %.3f'  #
            #       % (epoch + 1, train_l_sum / n, train_acc_sum / n))


    def getpara(self):
        print(self.w, self.b)


    def predict(self, X):
        input = X.view(-1, self.inputs_num)
        """前向传播过程"""
        for i, param_u in enumerate(self.params):  # MLP多层实现
            w, b = param_u
            activation = self.acti_list[i]

            if activation == 'relu':
                output = F.relu(tool.linreg(input, w, b))
            elif activation == 'tanh':
                output = F.tanh(tool.linreg(input, w, b))
            elif activation == 'selu':
                output = F.selu(tool.linreg(input, w, b))
            elif activation == 'sigmoid':
                output = F.sigmoid(tool.linreg(input, w, b))
            else:
                output = tool.linreg(input, w, b)
            input = output
        y_hat = input
        return y_hat

run.py (for scratch ‘fashion-mnist’)

#coding=utf-8
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
import numpy as np
import tool

from model import MyMLP

train_iter, test_iter = tool.load_data_fashion_mnist(50)
net = MyMLP(node_list=[784, 256, 10], acti_list=['relu', 'None'], classifier=True)
net.fit(train_iter, test_iter, num_epochs=20)
"""
由于fashion_mnist数据集有打包好的train_iter，而lending数据集没有设好的train_iter,所以lending的输入存在问题
"""

model.py(pytorch for lending)

#coding=utf-8
import torch
# from utils import load_dataset
import tool

class myMLP(torch.nn.Module):
    def __init__(self):
        super(myMLP, self).__init__()
        self.layer1 = torch.nn.Linear(9, 6)
        self.layer2 = torch.nn.Linear(6, 2)

    def forward(self, x):
        x = self.layer1(x)
        x = torch.nn.functional.relu(x)
        x = self.layer2(x)
        x = torch.nn.functional.softmax(x, dim=1)
        return x

    def run(self, X, y, epochs=20, lr=0.1):

        loss = torch.nn.CrossEntropyLoss()  # 不是函数 而是类
        optim = torch.optim.SGD(self.parameters(), lr=lr)  # 类的实现
        evaluate = tool.evaluate_classifier

        for i in range(epochs):
            train_l_sum, train_acc_sum, n = 0.0, 0.0, 0.0
            train_iter = tool.data_iter(50, X, y)
            for X, y in train_iter:
                optim.zero_grad()
                y_hat = self.forward(X)
                l = loss(y_hat, y)
                l.backward()
                optim.step()

                train_l_sum += l.item()
                train_acc_sum += evaluate(y_hat, y)
                n += y.shape[0]
            print('epoch %d, loss %.4f, train acc %.3f'
                  % (i + 1, train_l_sum / n, train_acc_sum / n))

run.py (for pytorch ‘lending’)

引入数据集需要anchor的utils文件

from utils import load_dataset

from model import myMLP
import torch
import tool

data_name = 'lending'
dataset_folder = 'D:\\Work\\work\\anchor2\\anchor-experiments-master\\datasets'
dataset = load_dataset(data_name, balance=True, dataset_folder=dataset_folder)

X = torch.tensor(dataset.train, dtype=torch.float)
y = torch.LongTensor(dataset.labels_train)
net = myMLP()

net.run(X, y)

model.py(pytorch for mnist)

#coding=utf-8
import torch
import tool

class myMLP(torch.nn.Module):
    def __init__(self, inputs_num):
        super(myMLP, self).__init__()
        self.inputs_num = inputs_num
        self.layer1 = torch.nn.Linear(self.inputs_num, 256)     # 更改节点数
        self.layer2 = torch.nn.Linear(256, 10)

    def forward(self, x):
        """
        input: sample=X[0], (1, 28, 28)
        output:  tensor([[1.9193e-12, 9.3560e-09, 1.1846e-06, 1.1439e-08, 3.9723e-08, 3.5939e-02,
                          1.0717e-07, 2.1095e-02, 9.6254e-05, 9.4287e-01]],grad_fn=<SoftmaxBackward>)
        
        """
        x = x.view(-1, self.inputs_num)
        x = self.layer1(x)
        x = torch.nn.functional.relu(x)
        x = self.layer2(x)
        x = torch.nn.functional.softmax(x, dim=1)
        return x
    
    
    def predict_dis(self, x):
        """
        input: sample=X[0], (1, 28, 28)
        output: tensor([9, ],grad_fn=<SoftmaxBackward>)   # forward的列维变行维
        
        for X, y in test_iter:
            sample = X[0:5]     # torch.Size([5, 1, 28, 28])
            s_y = y[:5]
            print(sample.shape)
            break
        net.predict_dis(sample) # tensor([9, 2, 1, 1, 2])
        s_y   # tensor([9, 2, 1, 1, 2])
        
        """
        x = x.view(-1, self.inputs_num)
        x = self.layer1(x)
        x = torch.nn.functional.relu(x)
        x = self.layer2(x)
        x = torch.nn.functional.softmax(x, dim=1)
        return x.argmax(dim=1)

    
    def run(self, train_iter, test_iter, epochs=20, lr=0.1):

        loss = torch.nn.CrossEntropyLoss()  # 不是函数 而是类
        optim = torch.optim.SGD(self.parameters(), lr=lr)  # 类的实现
        evaluate = tool.evaluate_classifier

        for epoch in range(epochs):
            train_l_sum, train_acc_sum, n = 0.0, 0.0, 0.0
#             train_iter = tool.data_iter(50, X, y)
            for X, y in train_iter:
#                 print(X.shape)
                optim.zero_grad()
                y_hat = self.forward(X)
                l = loss(y_hat, y)
                l.backward()
                optim.step()
            
                """评估"""
                train_l_sum += l.item()
                train_acc_sum += evaluate(y_hat, y)
                n += y.shape[0]
            test_acc = tool.evaluate_accuracy_for_softmax(test_iter, self.forward)
            print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
                  % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))

run.py (for pytorch mnist)

#coding=utf-8
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
import numpy as np
import tool

# from model import MyMLP

train_iter, test_iter = tool.load_data_fashion_mnist(50)
"""
由于fashion_mnist数据集有打包好的train_iter，而lending数据集没有设好的train_iter,所以lending的输入存在问题
"""
net = myMLP(784)
net.run(train_iter, test_iter, epochs=10)

for X, y in test_iter:
    sample = X[0:5]     # torch.Size([5, 1, 28, 28])
    s_y = y[:5]
    print(sample.shape)
    break
print(net.predict_dis(sample))