pytorch-常规操作

1，张量扩增(expand, repeat)

expand将tensor作为整体扩充（填入的是放大后的维度数，除非tensor的某个dim的shape=1，此时可做到同纬度扩充，否则只能升维），repeat也是将tensor做为整体扩充（填入的是放大的倍数，使用起来要灵活的多）

>>> a = torch.randn(2, 4)
>>> a
tensor([[-0.1346,  0.3429, -1.3040, -0.6949],
        [-0.0433,  1.7080, -1.8213, -1.6689]])
>>> a.expand(2,2,4)
tensor([[[-0.1346,  0.3429, -1.3040, -0.6949],
         [-0.0433,  1.7080, -1.8213, -1.6689]],

        [[-0.1346,  0.3429, -1.3040, -0.6949],
         [-0.0433,  1.7080, -1.8213, -1.6689]]])

>>> a.repeat(1,2)
tensor([[-0.1346,  0.3429, -1.3040, -0.6949, -0.1346,  0.3429, -1.3040, -0.6949],
        [-0.0433,  1.7080, -1.8213, -1.6689, -0.0433,  1.7080, -1.8213, -1.6689]])
>>> a.repeat(2,1,1)
tensor([[[-0.1346,  0.3429, -1.3040, -0.6949],
         [-0.0433,  1.7080, -1.8213, -1.6689]],

        [[-0.1346,  0.3429, -1.3040, -0.6949],
         [-0.0433,  1.7080, -1.8213, -1.6689]]])

整体扩充和交替扩充

>>> a = torch.randn(2,2)
>>> a
tensor([[ 0.2356,  0.0189],
        [-0.3703, -0.0547]])
>>> a.repeat(2,1)
tensor([[ 0.2356,  0.0189],
        [-0.3703, -0.0547],
        [ 0.2356,  0.0189],
        [-0.3703, -0.0547]])
>>> a.repeat(1,2).reshape(-1, a.shape[1])
tensor([[ 0.2356,  0.0189],
        [ 0.2356,  0.0189],
        [-0.3703, -0.0547],
        [-0.3703, -0.0547]])

注意，torch和numpy中的repeat效果不一致；numpy中有tile而torch中没有；numpy中的tile和torch中的repeat效果一致

'''numpy'''
>>> a = np.array([1,2,3,4])
>>> np.tile(a, 10)
array([1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2,
       3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4])
>>> a.repeat(10)
array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3,
       3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4])
>>> a.expand(10)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: 'numpy.ndarray' object has no attribute 'expand'
'''torch'''
>>> a = torch.arange(5)
>>> a
tensor([0, 1, 2, 3, 4])
>>> a.repeat(10)
tensor([0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3,
        4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2,
        3, 4])
>>> torch.tile(a, 10)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: module 'torch' has no attribute 'tile'

2，维度扩展（unsqueeze，切片）

>>> a = torch.randn(2)
>>> a
tensor([2.0488, 0.5997])
>>> b = a.unsqueeze(-1)
>>> b
tensor([[2.0488],
        [0.5997]])
>>> b.shape
torch.Size([2, 1])

>>> b = a[:, None]
>>> b
tensor([[2.0488],
        [0.5997]])
>>> b.shape
torch.Size([2, 1])

'''并且切片可以一次扩展多个维度'''
>>> b = a[None, :, None]
>>> b
tensor([[[2.0488],
         [0.5997]]])
>>> b.shape
torch.Size([1, 2, 1])

3，梯度取反（Function）

import torch
from torch.autograd import Function
import torch.nn as nn
import torch.optim as optim
from tqdm import tqdm
from matplotlib import pyplot as plt

class ReverseLayer(Function):
    @staticmethod
    def forward(ctx, x):
        return x

    @staticmethod
    def backward(ctx, grad_output):
        return grad_output.neg()


class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.parameter1 = nn.Parameter(torch.ones(10, 10))
        self.parameter2 = nn.Parameter(torch.ones(10, 10))
        self.parameter3 = nn.Parameter(torch.ones(10, 10))

    def forward(self, x):
        return x @ self.parameter1 @ self.parameter2 @ self.parameter3


class ReverseNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.parameter1 = nn.Parameter(torch.ones(10, 10))
        self.parameter2 = nn.Parameter(torch.ones(10, 10))
        self.parameter3 = nn.Parameter(torch.ones(10, 10))

    def forward(self, x):
        x1 = x @ self.parameter1
        x2 = ReverseLayer.apply(x1 @ self.parameter2)
        return x2 @ self.parameter3


dataInput = torch.randn(2, 10)
dataTarget = torch.randn(2, 10)

net1 = Net()
net2 = ReverseNet()
loss1 = torch.mean(net1(dataInput) - dataTarget)
loss1.backward()
loss2 = torch.mean(net2(dataInput) - dataTarget)
loss2.backward()
print('=======================PARAMETER1============================')
print(net1.parameter1.grad[0])
print(net2.parameter1.grad[0])
print('=======================PARAMETER2============================')
print(net1.parameter2.grad[0])
print(net2.parameter2.grad[0])
print('=======================PARAMETER3============================')
print(net1.parameter3.grad[0])
print(net2.parameter3.grad[0])
'''
It can be seen that due to the chain rule,
the derivative of all the layers before the reverse layer is taken to be negative
'''

optim1 = optim.Adam(net1.parameters())
optim2 = optim.Adam(net2.parameters())
loss1List = []
loss2List = []
epoch = 100
for i in tqdm(range(epoch)):
    net1.zero_grad()
    net2.zero_grad()

    loss1 = torch.mean(net1(dataInput) - dataTarget)
    loss1List.append(loss1.item())
    loss1.backward()
    optim1.step()

    loss2 = torch.mean(net2(dataInput) - dataTarget)
    loss2List.append(loss2.item())
    loss2.backward()
    optim2.step()


plt.subplot(2, 1, 1)
plt.plot(loss1List)
plt.subplot(2, 1, 2)
plt.plot(loss2List)
plt.show()
'''
It can be seen that 
Without reverselayer, loss decreases (min)
With reverselayer, the loss increases (max) 
'''

'''========================应用场景：网络拼接========================'''
'''========================不取反========================'''
import torch
import torch.nn as nn

myNet1 = nn.Linear(10, 10)
myNet2 = nn.Linear(10, 10)
loss = nn.PairwiseDistance(p=2)
optimizer = torch.optim.Adam(myNet1.parameters(), lr=1e-2)
epoch = 500
dataIn = torch.randn(1, 10)
dataOut = torch.ones(1, 10)
print(myNet2(myNet1(dataIn)))
for i in range(epoch):
    optimizer.zero_grad()
    l = loss(myNet2(myNet1(dataIn)), dataOut)
    l.backward()
    optimizer.step()
print(myNet2(myNet1(dataIn)))
'''========================应用：取反========================'''
import torch
import torch.nn as nn
from torch.autograd import Function

class ReverseLayerF(Function):
    @staticmethod
    def forward(ctx, x):
        return x

    @staticmethod
    def backward(ctx, grad_output):
        return grad_output.neg()

myNet1 = nn.Linear(10, 10)
myNet2 = nn.Linear(10, 10)
loss = nn.PairwiseDistance(p=2)
optimizer = torch.optim.Adam(myNet1.parameters(), lr=1e-2)
epoch = 500
dataIn = torch.randn(1, 10)
dataOut = torch.ones(1, 10)
print(myNet2(myNet1(dataIn)))
for i in range(epoch):
    optimizer.zero_grad()
    l = loss(myNet2(ReverseLayerF.apply(myNet1(dataIn))), dataOut)
    l.backward()
    optimizer.step()
print(myNet2(myNet1(dataIn)))

4，求梯度

'''v1'''
gradients = autograd.grad(outputs=dataOut, inputs=dataIn,
                              grad_outputs=torch.ones(dataIn.size()).cuda(),
                              create_graph=True, retain_graph=True, only_inputs=True)[0]
'''v2'''
dataOut.backward(torch.ones_like(dataOut))

5，CNN与LSTM输入输出维度含义

• CNN
  卷积data的四个维度: batch, input channel, height, width
  Conv2d的四个维度: input channel, output channel, kernel, stride

• LSTM
  时间序列data(输入)的三个维度: sequential-length(近似于NLP中一句话里几个单词), batch, input-size(一个单词几个字母)
  LSTM的三个维度: input-size, output-size, layers
  h0的三个维度: layers, batch, output-size
  c0的三个维度: layers, batch, output-size
  output的三个维度: sequential-length, batch, output-size

6，一维向量的转换-对角矩阵(diag)、one-hot标签(torch.nn.functional.one_hot)

转对角矩阵
diagonalMatrix = torch.diag(tensor)
转one-hot标签
torch.nn.functional.one_hot(tensor/Long Tensor, num_classes)

7，手动修改网络参数（load_state_dict）

• model.state_dict()返回的只是module类内部state dict对象的一个copy，如果只是在此拷贝上进行修改会发现对原先的model并不会有影响
• 解决方案是将model.state_dict() 字典赋值给给一个变量model_dict, 然后在model_dict上修改，最后model.load_state_dict(model_dict)

'''直接修改无变化'''
import torch
import torch.nn as nn

net = nn.Linear(10, 10)
optimizer = torch.optim.Adam(net.parameters(), lr=1e-2)
loss = nn.PairwiseDistance(p=2)
dataIn = torch.randn(2, 10)
dataOut = torch.ones(2, 10)
epoch = 200
for i in range(epoch):
    optimizer.zero_grad()
    l = loss(net(dataIn), dataOut).mean()
    l.backward()
    optimizer.step()
print(f'\033[33m{net(dataIn)}\033[0m')
for key in net.state_dict():
    print(net.state_dict()[key])
    net.state_dict()[key].data = torch.randn(net.state_dict()[key].shape)
for key in net.state_dict():
    print(net.state_dict()[key])
print(f'\033[34m{net(dataIn)}\033[0m')

'''先赋值再加载'''
import torch
import torch.nn as nn

net = nn.Linear(10, 10)
optimizer = torch.optim.Adam(net.parameters(), lr=1e-2)
loss = nn.PairwiseDistance(p=2)
dataIn = torch.randn(2, 10)
dataOut = torch.ones(2, 10)
epoch = 200
for i in range(epoch):
    optimizer.zero_grad()
    l = loss(net(dataIn), dataOut).mean()
    l.backward()
    optimizer.step()
print(f'\033[33m{net(dataIn)}\033[0m')
model_dict = net.state_dict()
for key in model_dict:
    print(model_dict[key])
    model_dict[key] = torch.randn(net.state_dict()[key].shape)
net.load_state_dict(model_dict)
for key in net.state_dict():
    print(net.state_dict()[key])
print(f'\033[34m{net(dataIn)}\033[0m')

8，显示模型结构的方法

'''显示模块'''
myNet
list(myNet.children())
'''显示参数名称及数值'''
mynet.state_dict().keys()
mynet.state_dict()
'''显示参数数值'''
list(mynet.parameters())

9，显示前k个最大值的索引

'''k=1'''
tensor.argmax()
'''k>1'''
tansor.argsort()[:k]

10，打乱(tensor[torch.randperm(tensor.shape[0])])

dataRandomIndex = torch.randperm(data.shape[0])
data[dataRandomIndex]

11，可视化：特征图(feture map)、卷积核权重、卷积核最匹配样本、类别激活图(Class Activation Map/CAM)、网络结构

https://www.cnblogs.com/tensorzhang/p/15053885.html

12，不使用optim进行训练的步骤

for i in range(epoch):
  data = nn.Parameter()
  l = loss
  l.backward()
  data = data - lr*data.grad
  data = nn.Parameter(data.detach())

13，生成one-hot标签 nn.funcitonal.one_hot(label, nums_classes=N)

oneHotLabel = torch.nn.functional.ones_hot(label, num_classes=N).to(torch.float)
'''label从0开始'''
np.eye(num_classes)[arr]
np.eye(7)[np.ones(10, dtype=np.int)]

14，GPU自动匹配gpu = torch.device(f’cuda:0’ if torch.cuda.is_available() else ‘cpu’)

15，矩阵乘法

a * b，要求两个矩阵维度完全一致，即两个矩阵对应元素相乘，输出的维度也和原矩阵维度相同
torch.mul(a, b)是矩阵a和b对应位相乘，a和b的维度必须相等，比如a的维度是(1, 2)，b的维度是(1, 2)，返回的仍是(1, 2)的矩阵

a@b，要求两个矩阵维度是（n×m）和（m×p），即普通二维矩阵乘法
torch.mm(a,b)，要求两个矩阵维度是（n×m）和（m×p），即普通二维矩阵乘法

torch.matul(a,b)，matmul可以进行张量乘法，输入可以是高维，当输入是多维时，把多出的一维作为batch提出来，其他部分做矩阵乘法
当输入是二维矩阵时，torch.mm(a,b)和torch.matul(a,b)是一样的

参考博客：https://blog.csdn.net/lijiaming_99/article/details/114642093