【Pytorch】Pytorch 建模常用总结

1. 提取模型中间层输出(hook, .register_forward_hook(hook=hook))

1.1 定义模型

import torch
import torch.nn as nn

class StudentModel(nn.Module):
    def __init__(self, input=25, output=2):
        super().__init__()
        self.l1 = nn.Linear(input, 16)
        self.l2 = nn.Linear(16, 32)
        self.l3 = nn.Linear(32, 2)
        self.relu = nn.ReLU()
        self.softmax = nn.Softmax(dim=1)
    
    def forward(self, X):
        x = X.reshape(-1, 25)
        x = self.relu(self.l1(x))
        x = self.relu(self.l2(x))
        x = self.relu(self.l3(x))

        x = self.softmax(x)
        res = x.max(1).indices
        return torch.tensor(res, dtype=float)

sm = StudentModel()
'''
StudentModel(
  (l1): Linear(in_features=25, out_features=16, bias=True)
  (l2): Linear(in_features=16, out_features=32, bias=True)
  (l3): Linear(in_features=32, out_features=2, bias=True)
  (relu): ReLU()
  (softmax): Softmax(dim=1)
)
'''

1.2 定义hook函数 调用.register_forward_hook(hook=hook)

features_in_hook = []
features_out_hook = []

def hook(module, fea_in, fea_out):
    features_in_hook.append(fea_in)
    features_out_hook.append(fea_out)
    return None

layer = sm.l1
layer.register_forward_hook(hook=hook)
'''
<torch.utils.hooks.RemovableHandle at 0x11b18a5b0>
'''

1.3 模型跑数据

sm(Xtrain)
'''
tensor([0., 1., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
        1., 1., 1., 0., 1., 1., 1., 1., 0., 1., 1., 1., 1., 1., 1., 1., 1., 0.,
        1., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 1., 1.,
        1., 1., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
        1., 1., 1., 1., 0., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 1., 0., 1.,
        1., 1., 1., 0., 1., 1., 1., 1., 1., 1.], dtype=torch.float64)
'''

1.4 获取中间层输出

features_in_hook[0][0].shape
'''
torch.Size([100, 25])
'''
features_in _hook
'''
[(tensor([[0.4921, 0.8465, 0.3133,  ..., 0.8547, 0.0275, 0.0872],
          [0.1217, 0.5101, 0.9094,  ..., 0.0517, 0.5217, 0.8201],
          [0.3493, 0.6255, 0.7081,  ..., 0.6097, 0.3401, 0.0455],
          ...,
          [0.6717, 0.0128, 0.2326,  ..., 0.1152, 0.5885, 0.9372],
          [0.8665, 0.8432, 0.3781,  ..., 0.3370, 0.8750, 0.4666],
          [0.7063, 0.1330, 0.3501,  ..., 0.5658, 0.2588, 0.3254]],
         grad_fn=<ReshapeAliasBackward0>),)]
'''

features_out_hook[0].shape
'''
torch.Size([100, 16])
'''
features_out_hook
'''
tensor([[ 0.1448, -0.0800, -0.1307,  ..., -0.1472,  0.6660, -0.0465],
        [ 0.4460,  0.2587, -0.1345,  ..., -0.4809,  0.8669, -0.2805],
        [ 0.2118,  0.0445, -0.1910,  ..., -0.5179,  0.9163, -0.0972],
        ...,
        [ 0.1761, -0.0178, -0.4799,  ..., -0.3769,  0.6353, -0.3738],
        [ 0.1534, -0.0314, -0.2041,  ..., -0.3669,  1.0264, -0.1857],
        [-0.0438,  0.0639, -0.2547,  ..., -0.4396,  0.7633, -0.2492]],
       grad_fn=<AddmmBackward0>)
'''

2. 插值处理 上下采样 torch.nn.functional.interpolate()

import torch.nn.functional as F

'''
mode 
	"nearest" 复制最近的一个元素实现插值，输入必须4维
	"linear" 线性插值
	...
'''

x = torch.rand(1, 1, 3, 4)
F.interpolate(x, (5, 5), mode="nearest").shape
'''
torch.Size([1, 1, 5, 5])
'''

x = torch.rand(1, 3, 3)
F.interpolate(x, 5, mode="linear").shape
'''
torch.Size([1, 3, 5])
'''

3. 指定维度求最大值索引 torch.max(x, dim=).indices

x = torch.rand(3, 2)
'''
tensor([[0.5832, 0.0757],
        [0.4064, 0.4217],
        [0.1081, 0.5463]])
'''
torch.max(x, dim=1) 
'''
每行的最大值及其索引
torch.return_types.max(
values=tensor([0.5832, 0.4217, 0.5463]),
indices=tensor([0, 1, 1]))
'''
torch.max(x, dim=1).indices
'''
适合二分类
tensor([0, 1, 1]))
'''

4. 模型转GPU计算

需要转换的对象

• 模型
• 损失函数
• 数据（特征数据、标签）

4.1 .cuda()

'''
.cuda(0) 指定第0块GPU
'''
if torch.cuda.is_available():
    model = model.cuda()
    loss_fn = loss_fn.cuda()
    X = X.cuda()
    y = y.cuda()
    

4.2 .to(device)

# 第二种方法
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
device0 = torch.device('cuda:0')

model.to(device)
loss_fn.to(device)
X = X.to(device)
y = y.to(device)

4.3 多GPU并行计算

4.3.1 单进程多GPU训练(DP)模式 torch.nn.DataParallel

• 并行的多卡都是由一个进程进行控制，在进行梯度的传播时，是在主GPU上进行的。

1. 将模型布置到多个指定GPU上
   model = torch.nn.DataParallel(model,device_ids=device_list)
2. 指定模型布置的主GPU
   main_dev = 0
model.to(main_dev)
3. 优化器放到主GPU
   optimizer = optim.Adam(model.parameters(), lr=0.001)
optimizer.to(device)
4. 数据放到主GPU
   data.to(main_dev)

device_list = [0, 1, 2, 3]
main_dev = device_list[0]  #主GPU，也就是分发任务和结果回收的GPU，也是梯度传播更新的GPU
model = torch.nn.DataParallel(model,device_ids=device_list)
model.to(main_dev)
 
for data in train_dataloaders: 
   model.train(True)
   inputs, labels = data
   inputs = Variable(inputs.to(main_dev))  #将数据放到主要GPU
   labels = Variable(labels.to(main_dev)) 

4.4 限定GPU可用

import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1,2"

# 从可用GPU中搜索第0和1的GPU
Model = nn.DataParallel(Model, device_ids=[0,1])

5. 模型更改、添加和删除层

5.1 model.add_module()

m = Treelinears()
m
'''
TreeLinears(
  (l1): Linear(in_features=8, out_features=4, bias=True)
  (l2): Linear(in_features=4, out_features=2, bias=True)
  (l3): Linear(in_features=2, out_features=1, bias=True)
)
'''

m.add_module('add_l4', nn.Linear(1, 1))
m
'''
TreeLinears(
  (l1): Linear(in_features=8, out_features=4, bias=True)
  (l2): Linear(in_features=4, out_features=2, bias=True)
  (l3): Linear(in_features=2, out_features=1, bias=True)
  (add_l4): Linear(in_features=1, out_features=1, bias=True)
)
'''

5.2 Sequential().add_module()

sq = nn.Sequential()
sq.add_module('add_l1', nn.Linear(1, 1))
sq.add_module('add_l2', nn.Linear(1, 1))
'''
Sequential(
  (add_l1): Linear(in_features=1, out_features=1, bias=True)
  (add_l2): Linear(in_features=1, out_features=1, bias=True)
)
'''

5.3 del model.layer

del sq.add_l1
'''
Sequential(
  (add_l2): Linear(in_features=1, out_features=1, bias=True)
)
'''

6. 获取模型参数量、FLOPs

6.1 thop.profile

import torch
from thop import profile

def get_flops_params_idx4(model, input_shape=[256, 1, 28, 28]):
    """
    :param model: 传入模型
    :param inshape: 模型的输入shape
    :return : (flops, params)
    """
    tensor = (torch.rand(*inpust_shape), )
    flops, params = profile(model, tensor) # 更改源码 不print

    return flops, params

6.2 torch 自带接口计算参数量

# 参数量
sum([param.nelement() for param in model.parameters()])

7. 损失函数

7.1 均方误差 torch.nn.MSELoss()

import torch
import torch.nn as nn

# 预测是[0,1,2]的概率
X = [[0.1, 0.8, 0.3],
     [0.2, 0.9, 0.1]]
X = torch.tensor(X)
# 真实标签
y = [[0], [0]]
y = torch.tensor(y)
'''
均方误差，多用于回归
X输入维度必须为(N, C)，N样本数，C类别数
y输入维度必须为(N, 1), N样本数，第二维度1填写真实标签值
'''
nn.MSELoss()(X, y)
'''
tensor(0.2667)
'''

7.2 交叉熵损失 torch.nn.CrossEntropyLoss()

nn.CrossEntropyLoss 在内部执行了 softmax 操作，不需要显式地在模型的最后一层添加 softmax 激活函数。

import torch
import torch.nn as nn

X = torch.rand(10, 8)
y = torch.randint(0, 2, (10, ))
'''
交叉熵损失，多用于多分类
X输入维度必须为(N, C)，N样本数，C类别数, dtype=float
y输入维度必须为(N, ), 第一维度N为样本数，填写真实标签值, dtype=long
'''
nn.CrossEntropyLoss()(X, y)
'''
tensor(1.0546)
'''

7.3 torch.nn.NULLLoss()

• 输入输出与nn.CrossEntropyLoss()一样，

• 区别在于: 内部没有softmax操作，需要显示调用torch.log_softmax(x, dim=1), 再输入nn.NULLLoss()。

7.4 torch.nn.BCELoss()/BCEWithLogitsLoss()

解决二分类问题，模型输出层设置为nn.Linear(n, 1)

1. BCELoss()(input, target) 解决二分类问题，模型输出层应接nn.sigmoid() ；
   input 维度不定，target维度与input维度相同，值为0or1；

2. BCEWithLogitsLoss() >>> nn.sigmoid + nn.BCELoss

8. 二分类 概率->索引

x = torch.tensor([[1, 2],
                  [3, 2]])

torch.max(x, 0)
'''
torch.return_types.max(
values=tensor([3, 2]),
indices=tensor([1, 0]))
'''
torch.max(x, 0).indices
'''
tensor([1, 0])
'''

9. pickle 保存读取数据/模型

import pickle

dic = {'a': [1, 2], 
       'b':[1, 2, 3]}

with open('dic.pkl', 'wb') as fp:
    pickle.dump(dic, fp)

with open('dic.pkl', 'rb') as fp:
    d = pickle.load(fp)

或

import pickle

fp = open('dic.pkl', 'wb')
pickle.dump(cols, fp)
fp.close()

fp = open('dic.pkl', 'rb')
data = pickle.load(fp)
fp.close()