Neural network classification task

### Torch install method
- CPU:pip install torch -i https://download.pytorch.org/shl/torch_stable.html
- GPU:CUDA and corresponding torch version
- http://download.pytorch.org/whl/torch_stable.html
- https://developer.nvidia.com/cuda-toolkit-archive

### Mnist sorting task:
   - Basic network construction and training methods
   - torch.nn.functional module
   - nn.Module module

### Read Mnist dataset
Automatic loading dataset

import torch
print(torch.__version__)

%matplotlib inline

from pathlib import Path
import requests

DATA_PATH = Path("data")
PATH = DATA_PATH / "mnist"

PATH.mkdir(parents=True, exist_ok=True)

URL = "https://github.com/mnielsen/neural-networks-and-deep-learning/tree/master/data"
FILENAME = "mnist.pkl.gz"

if not (PATH / FILENAME).exists():
    content = requests.get(URL + FILENAME).content
    (PATH / FILENAME).open("wb").write(content)

import pickle
import gzip

with gzip.open((PATH / FILENAME).as_posix(), "rb") as f:
    ((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f,encoding='latin-1')

x_train.shape

784 is the number of pixels per sample in minist dataset

from matplotlib import pyplot
import numpy as np

pyplot.imshow(x_train[0].reshape(28,28),cmap="gray")
print(x_train.shape)

Note that the data needs to be transformed into tensor to participate in the subsequent modeling training

import torch

x_train, y_train, x_valid, y_valid = map(
    torch.tensor, (x_train, y_train, x_valid, y_valid)
)
n, c = x_train.shape
x_train, x_train.shape, y_train.min(), y_train.max()
print(x_train, y_train)
print(x_train.shape)
print(y_train.min(), y_train.max())

### torch.nn.functional Many layers and functions are used here
torch.nn.functional have many features will be used. When to use nn.Module and when to use nn.functional? In general, If the model has learnable parameters, it is best to use nn.Module, then the other case nn.functional is relatively simple

# nn.function suitable for testing , not suitable for building actual modules
import torch.nn.functional as F

loss_func = F.cross_entropy

def model(xb):
    return xb.mm(weights) + bias # mm representation matris multiplication

bs =64
xb = x_train[0:bs]
yb = y_train[0:bs]
weights = torch.randn([784, 10], dtype = torch.float, requires_grad = True)
bs =64
bias = torch.zeros(10, requires_grad = True)

print(loss_func(model(xb), yb))

### Creat a module to simplify the code
- You must inherit mm.Module and call the constructor of nn.Module in its constructor
- Instead of writing backpropagators, nn.Module can automatically implement backpropagation using autograd
- A learnable parameter in a module can return an iterator by either named_parameters() or parameters()

from torch import nn

class Mnist_NN(nn.Module):
    def __init__(self):
        super().__init__()
        self.hidden1 = nn.Linear(784, 128)
        self.hidden2 = nn.Linear(128, 256)
        # self.hidden3 = nn.Linear(256, 512)
        self.out = nn.Linear(256, 10)
        self.dropout = nn.Dropout(0.5)
        
    def forward(self, x):
        x = F.relu(self.hidden1(x))
        x = self.dropout(x)
        x = F.relu(self.hidden2(x))
        x = self.dropout(x)
        # x = F.relu(self.hidden3(x))
        x = self.out(x)
        
        return x

from torch import nn

class Mnist_NN(nn.Module):
    def __init__(self):
        super().__init__()
        self.hidden1 = nn.Linear(784, 128)
        self.hidden2 = nn.Linear(128, 256)
        self.hidden3 = nn.Linear(256, 512)
        self.out = nn.Linear(512, 10)
        self.dropout = nn.Dropout(0.5)
        
    def forward(self, x):
        x = F.relu(self.hidden1(x))
        x = self.dropout(x)
        x = F.relu(self.hidden2(x))
        x = self.dropout(x)
        x = F.relu(self.hidden3(x))
        x = self.dropout(x)
        x = self.out(x)
        
        return x

net = Mnist_NN()
print(net)

We can print the weights and bias items in our defined names

for name, parameter in net.named_parameters():
    print(name, parameter, parameter.size())

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)

### Use TensorDataset and DataLoder to simplify the construction of datasets

from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader

train_ds = TensorDataset(x_train, y_train)
train_dl = DataLoader(train_ds,batch_size = bs , shuffle = True)

valid_ds = TensorDataset(x_valid, y_valid)
valid_dl = DataLoader(valid_ds,batch_size = bs * 2)

def get_data(train_ds, valid_ds, bs):
    return(
        DataLoader(train_ds, batch_size = bs, shuffle = True),
        DataLoader(valid_ds, batch_size = bs *2)
    )

- It is common to add model.train() when training the module, so that Batch Normalization and Dropout are used normally
- model.eval() is generally selected during testing, so that Batch Normalization and Dropout are not used

import numpy as np

def fit(steps, model, loss_func, opt, train_dl, valid_dl):
    for step in range(steps):
        model.train()
        for xb, yb in train_dl:
            loss_batch(model, loss_func, xb, yb, opt)
            
        model.eval()
        with torch.no_grad():
            losses, nums = zip(
                *[loss_batch(model, loss_func, xb, yb) for xb, yb in valid_dl]
            )
        val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)
        print("Current step:" + str(step), "Verification set loss:" + str(val_loss))

### zip usage

a = [1,2,3]
b = [4,5,6]
zipped = zip(a,b)
print(list(zipped))
a2, b2 = zip(*zip(a,b))
print(a2)
print(b2)

from torch import optim

def get_model():
    model = Mnist_NN()
    return model, optim.Adam(model.parameters(), lr = 0.001) # Adam

def loss_batch(model, loss_func, xb, yb, opt=None):
    loss = loss_func(model(xb), yb)
    
    if opt is not None:
        loss.backward() # backpropagation
        opt.step() # update parameters
        opt.zero_grad() # gradient clearing
    
    return loss.item(), len(xb)

### Three line complete!

train_dl, valid_dl = get_data(train_ds, valid_ds, bs)
model, opt = get_model()
fit(100, model, loss_func, opt, train_dl, valid_dl)

### practice
- Change the optimizer from SGD to Adam
- Change the number of layers and the number of neurons to observe the effect
- Calculate the accuracy of the current model

correct = 0
total = 0
for xb, yb in valid_dl:
    outputs = model(xb)
    _, predicted = torch.max(outputs.data, 1) # Max value and Index
    total += yb.size(0)
    correct += (predicted == yb).sum().item() # .item Transform tensor into ndarray,easy drawing
    
print("Accuracy of the network on the 10000 test images: %d %%" % (100 * correct / total))