defforward(self, x):
out = F.relu(self.linear_1(x))# During training, randomly zeroes some of the elements of the input# tensor with probability :attr:`p` using samples from a Bernoulli# distribution.# Dropout说的简单一点就是:我们在前向传播的时候,让某个神经元的激活值以一定的概率p停止工作
out = F.dropout(out, p=dropout_prob, training=True)
out = F.relu(self.linear_2(out))
out = F.dropout(out, p=dropout_prob, training=True)
logits = self.linear_out(out)
probas = F.log_softmax(logits, dim=1)return logits, probas
5. GNN with Gaussian model
from scipy.spatial.distance import cdist
from torch.utils.data.dataset import Subset
import time
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import datasets
from torchvision import transforms
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt
# Device
DEVICE = torch.device("cuda:0"if torch.cuda.is_available()else"cpu")# Hyperparameters
RANDOM_SEED =1
LEARNING_RATE =0.05
NUM_EPOCHS =2
BATCH_SIZE =128
IMG_SIZE =28# Architecture
NUM_CLASSES =10
train_indices = torch.arange(0,59000)
valid_indices = torch.arange(59000,60000)
custom_transform = transforms.Compose([transforms.ToTensor()])
train_and_valid = datasets.MNIST(root='data',train=True, transform=custom_transform,download=True)
test_dataset = datasets.MNIST(root='data',train=False,transform=custom_transform,download=True)
train_dataset = Subset(train_and_valid, train_indices)
valid_dataset = Subset(train_and_valid, valid_indices)
train_loader = DataLoader(dataset=train_dataset,batch_size=BATCH_SIZE,num_workers=4,shuffle=True)
valid_loader = DataLoader(dataset=valid_dataset,batch_size=BATCH_SIZE,num_workers=4,shuffle=False)
test_loader = DataLoader(dataset=test_dataset,batch_size=BATCH_SIZE,num_workers=4,shuffle=False)defprecompute_adj_matrix(img_size):# col:28*28, row:28*28# col = [[0 1 2 3 ... n], [0 1 2 3 ... n], ... [0 1 2 3 ... n]]# row = [[0 0 0 0 ... 0], [1 1 1 ... n], [n n n ... n]]
col, row = np.meshgrid(np.arange(img_size), np.arange(img_size))# N = img_size^2# construct 2D coordinate array (shape N * 2) and normalize in range [0, 1]# >>> arrays = [np.random.randn(3, 4) for _ in range(10)]# >>> np.stack(arrays, axis=0).shape# (10, 3, 4)## >>> np.stack(arrays, axis=1).shape# (3, 10, 4)## >>> np.stack(arrays, axis=2).shape# (3, 4, 10)#
coord_without_norm = coord = np.stack((col, row), axis=2).reshape(-1,2)# coord: 784*2
coord = np.stack((col, row), axis=2).reshape(-1,2)/ img_size
# compute pairwise distance matrix (N x N)# Find the Euclidean distances between four 2-D coordinates:## >>> from scipy.spatial import distance# >>> coords = [(35.0456, -85.2672),# ... (35.1174, -89.9711),# ... (35.9728, -83.9422),# ... (36.1667, -86.7833)]# >>> distance.cdist(coords, coords, 'euclidean')# array([[ 0. , 4.7044, 1.6172, 1.8856],# [ 4.7044, 0. , 6.0893, 3.3561],# [ 1.6172, 6.0893, 0. , 2.8477],# [ 1.8856, 3.3561, 2.8477, 0. ]])# dist: 784*784
dist = cdist(coord, coord, metric="euclidean")# Apply Gaussian filter
sigma =0.05*np.pi
A = np.exp(- dist / sigma**2)
A[A <0.01]=0# A:784*784
A = torch.from_numpy(A).float()# Normalization as per# D: 784
D = A.sum(1)# nodes degree (N,)
D_hat =(D +1e-5)**(-0.5)
A_hat = D_hat.view(-1,1)* A * D_hat.view(1,-1)# N,Nreturn A_hat
classGraphNet(nn.Module):def__init__(self, img_size=28, num_classes=10):super(GraphNet, self).__init__()
n_rows = img_size**2
self.fc = nn.Linear(n_rows, num_classes, bias=False)
A = precompute_adj_matrix(img_size)
self.register_buffer('A', A)defforward(self, x):
B = x.size(0)# Reshape Adjacency Matrix# [N, N] Adj. matrix -> [1, N, N] Adj tensor where N = H*W
A_tensor = self.A.unsqueeze(0)# [1, N, N] Adj tensor -> [B, N, N] tensor
A_tensor = self.A.expand(B,-1,-1)# Reshape inputs# [B, C, H, W] => [B, H*W, 1]
xShape_before_view = x.shape
x_reshape = x.view(B,-1,1)
xShape_after_view = x_reshape
# bmm = batch matrix product to sum the neighbor features# Input: [B, N, N] * [B, N, 1]# Output: [B, N]
avg_neighbor_features =(torch.bmm(A_tensor, x_reshape).view(B,-1))
logits = self.fc(avg_neighbor_features)
probas = F.softmax(logits, dim=1)return logits, probas
torch.manual_seed(RANDOM_SEED)
model = GraphNet(img_size=IMG_SIZE, num_classes=NUM_CLASSES)
model = model.to(DEVICE)
optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)defcompute_acc(model, data_loader, device):
correct_pred, num_examples =0,0for features, targets in data_loader:
features = features.to(device)
targets = targets.to(device)
logits, probas = model(features)
_, predicted_labels = torch.max(probas,1)
num_examples += targets.size(0)
correct_pred +=(predicted_labels == targets).sum()return correct_pred.float()/ num_examples *100
start_time = time.time()
cost_list =[]
train_acc_list, valid_acc_list =[],[]for epoch inrange(NUM_EPOCHS):
model.train()for batch_idx,(features, targets)inenumerate(train_loader):
features = features.to(DEVICE)
targets = targets.to(DEVICE)### FORWARD AND BACK PROP
logits, probas = model(features)
cost = F.cross_entropy(logits, targets)
optimizer.zero_grad()
cost.backward()### UPDATE MODEL PARAMETERS
optimizer.step()#################################################### CODE ONLY FOR LOGGING BEYOND THIS POINT################################################
cost_list.append(cost.item())ifnot batch_idx %150:print(f'Epoch: {epoch + 1:03d}/{NUM_EPOCHS:03d} | '
f'Batch {batch_idx:03d}/{len(train_loader):03d} |'
f' Cost: {cost:.4f}')
model.eval()with torch.set_grad_enabled(False):# save memory during inference
train_acc = compute_acc(model, train_loader, device=DEVICE)
valid_acc = compute_acc(model, valid_loader, device=DEVICE)print(f'Epoch: {epoch + 1:03d}/{NUM_EPOCHS:03d}\n'
f'Train ACC: {train_acc:.2f} | Validation ACC: {valid_acc:.2f}')
train_acc_list.append(train_acc)
valid_acc_list.append(valid_acc)
elapsed =(time.time()- start_time)/60print(f'Time elapsed: {elapsed:.2f} min')
elapsed =(time.time()- start_time)/60print(f'Total Training Time: {elapsed:.2f} min')
plt.plot(cost_list, label='Minibatch cost')
plt.plot(np.convolve(cost_list,
np.ones(200,)/200, mode='valid'),
label='Running average')
plt.ylabel('Cross Entropy')
plt.xlabel('Iteration')
plt.legend()
plt.show()
1. Perceptronimport numpy as npimport matplotlib.pyplot as pltimport torchimport osbasedir = os.path.dirname(os.path.realpath(__file__))# np.genfromtxt():Load data from a text file, with missing values handled as specified.data = np.genfromtxt(bas