块和层
块:可以描述单个层、由多个层组成的组件或整个模型本身。使用块进行抽象可以将一些块组成更大的组件。
在编程角度,块由类表示。它的任何子类都必须定义一个将其输入转化为输出的前向传播函数,并且必须存储任何必须的参数。
先看MLP的代码:
import torch
from torch import nn
net = nn.Sequential(nn.Linear(20,256),nn.ReLU(),nn.Linear(256,10))
x = torch.rand(2,20)
net(x) #net.__call__(x)的简写
tensor([[ 0.2305, 0.0688, -0.0887, 0.0833, 0.1858, 0.0170, -0.0810, 0.1837,
-0.2655, -0.0071],
[ 0.0535, 0.0221, -0.0006, -0.0011, 0.0999, -0.0787, -0.1264, 0.0393,
-0.1978, -0.0057]], grad_fn=<AddmmBackward>)
自定义块
from torch.nn import function as F
#自定义快
class MLP(nn.Module):
def __init__(self):
super(MLP, self).__init__()
self.linear1 = nn.Linear(20, 256)
self.linear2 = nn.Linear(256, 10)
def forward(self,x):
x = self.linear1(x)
x = F.relu(x)
x = self.linear2(x)
return x
#实例化
mlp = MLP()
print(mlp(x))
tensor([[ 0.0425, -0.1575, 0.0146, -0.4185, -0.2152, 0.1353, -0.1584, 0.0866,
0.0052, -0.3651],
[-0.0027, -0.0181, -0.0613, -0.2916, -0.0459, 0.1007, -0.1310, 0.0546,
0.1019, -0.2634]], grad_fn=<AddmmBackward>)
顺序块
#顺序块
class MySequential(nn.Module):
def __init__(self,*args):
super(MySequential, self).__init__()
for idx,module in enumerate(args):
self._modules[str(idx)] = module
#变量_modules中。module的类型是OrderedDict
def forward(self,x):
for block in self._modules.values():
x = block(x)
return x
#实例化
net = MySequential(nn.Linear(20,256),nn.ReLU(),nn.Linear(256,10))
print(net(x))
tensor([[ 0.1311, -0.0544, -0.0665, -0.0638, 0.1783, 0.0375, 0.0173, 0.0048,
-0.0642, -0.0604],
[ 0.1811, -0.0357, -0.0488, -0.1271, 0.0690, 0.0487, -0.0540, -0.0443,
-0.1325, -0.0720]], grad_fn=<AddmmBackward>)
enumerate() 函数用于将一个可遍历的数据对象(如列表、元组或字符串)组合为一个索引序列,同时列出数据和数据下标,一般用在 for 循环当中。
enumerate()函数的具体用法介绍
在前向传播函数中执行代码
#在前向传播函数中执行代码
class FixedHiddenMLP(nn.Module):
def __init__(self):
super(FixedHiddenMLP, self).__init__()
#不计算梯度的随即权重参数,因此在其训练期间保持不变
self.rand_weight = torch.rand((20,20),requires_grad = False)
self.linear = nn.Linear(20,20)
def forward(self,x):
x = self.linear(x)
#使用创建的常量参数以及relu和mm函数
x = F.relu(torch.mm(x,self.rand_weight)+1)
#复用全连接层。这相当于两个全连接层共享参数
x = self.linear(x)
#控制流
while x.abs().sum() > 1:
x/=2
return x.sum()
#实例化
net = FixedHiddenMLP()
print(net(x))
tensor(0.0136, grad_fn=<SumBackward0>)
块的混搭
#混搭块
class NestMLP(nn.Module):
def __init__(self):
super(NestMLP, self).__init__()
self.net = nn.Sequential(nn.Linear(20,64),nn.ReLU(),nn.Linear(64,32),nn.ReLU())
self.linear = nn.Linear(32,16)
def forward(self,x):
x = self.net(x)
x = self.linear(x)
return x
#实例化
net = nn.Sequential(NestMLP(),nn.Linear(16,20),FixedHiddenMLP())
print(net(x))
tensor(-0.2206, grad_fn=<SumBackward0>)
总结:
- 一个块可以由许多层组成,一个块也可以由许多块组成
- 块可以包含代码
- 块负责大量的内部处理,包含参数初始化和反向传播
- 层和块的顺序连接由Sequential块处理
参数访问
引入模型
import torch
from torch import nn
net = nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,1))
x = torch.rand(size=(2,4))
print(x)
print(net(x))
tensor([[0.5786, 0.0265, 0.9286, 0.4053],
[0.0824, 0.2364, 0.3520, 0.9443]])
tensor([[0.2221],
[0.2655]], grad_fn=<AddmmBackward>)
参数访问
#参数访问
print(net[2].state_dict()) #第二个全连接层的参数
print(type(net[2].bias))
print(net[2].bias)
print(net[2].bias.data)
OrderedDict([('weight', tensor([[ 0.3187, -0.3033, 0.3075, 0.1933, -0.0262, 0.1024, -0.2199, 0.2016]])), ('bias', tensor([-0.3335]))])
<class 'torch.nn.parameter.Parameter'>
Parameter containing:
tensor([-0.3335], requires_grad=True)
tensor([-0.3335])
一次性访问全部参数
#一次性访问全部参数
print(*[(name,param.shape) for name,param in net[0].named_parameters()])
print(*[(name,param.shape) for name,param in net.named_parameters()])
('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))
('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))
从嵌套块收集参数
#从嵌套块收集参数
def block1():
return nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,4),nn.ReLU())
def block2():
net = nn.Sequential()
for i in range(4):
net.add_module(f'block {i}',block1())
return net
rgnet = nn.Sequential(block2(),nn.Linear(4,1))
print(rgnet(x))
tensor([[-0.5766],
[-0.5767]], grad_fn=<AddmmBackward>)
参数管理
print(rgnet)
print(rgnet[0][1][0].bias.data) #访问第一个主要的块中、第二个子块的第一层的偏置项
Sequential(
(0): Sequential(
(block 0): Sequential(
(0): Linear(in_features=4, out_features=8, bias=True)
(1): ReLU()
(2): Linear(in_features=8, out_features=4, bias=True)
(3): ReLU()
)
(block 1): Sequential(
(0): Linear(in_features=4, out_features=8, bias=True)
(1): ReLU()
(2): Linear(in_features=8, out_features=4, bias=True)
(3): ReLU()
)
(block 2): Sequential(
(0): Linear(in_features=4, out_features=8, bias=True)
(1): ReLU()
(2): Linear(in_features=8, out_features=4, bias=True)
(3): ReLU()
)
(block 3): Sequential(
(0): Linear(in_features=4, out_features=8, bias=True)
(1): ReLU()
(2): Linear(in_features=8, out_features=4, bias=True)
(3): ReLU()
)
)
(1): Linear(in_features=4, out_features=1, bias=True)
)
tensor([ 0.0760, 0.1510, 0.2018, 0.1264, -0.4960, 0.3398, -0.0903, 0.0276])
内置参数初始化
#将所有权重参数初始化为标准差为0.01的高斯随机变量,且将偏置参数置为0
def init_normal(m):
if type(m) == nn.Linear:
nn.init.normal_(m.weight,mean=0,std=0.01)
nn.init.zeros_(m.bias)
net.apply(init_normal)
print(net[0].weight.data[0],net[0].bias.data[0])
#将所有参数初始化为给定的常数,例如1
def init_constant(m):
if type(m) == nn.Linear:
nn.init.constant_(m.weight,1)
nn.init.zeros_(m.bias)
net.apply(init_constant)
print(net[0].weight.data[0],net[0].bias.data[0])
#对某些块应用不同的初始化方法
def xavier(m):
if type(m) == nn.Linear:
nn.init.xavier_uniform_(m.weight)
def init_42(m):
if type(m) == nn.Linear:
nn.init.constant_(m.weight,42)
net[0].apply(xavier)
net[2].apply(init_42)
print(net[0].weight.data[0])
print(net[2].weight.data)
tensor([-0.0013, 0.0090, 0.0025, -0.0163]) tensor(0.)
tensor([1., 1., 1., 1.]) tensor(0.)
tensor([ 0.5687, -0.2902, -0.1938, 0.4916])
tensor([[42., 42., 42., 42., 42., 42., 42., 42.]])
自定义参数初始化
#自定义初始化
def my_init(m):
if type(m) == nn.Linear:
print(
"Init",
*[(name, param.shape) for name, param in m.named_parameters()][0])
nn.init.uniform_(m.weight, -10, 10)
m.weight.data *= m.weight.data.abs() >= 5
net.apply(my_init)
print(net[0].weight[:2])
Init weight torch.Size([8, 4])
Init weight torch.Size([1, 8])
tensor([[-0.0000, -9.5939, 0.0000, 0.0000],
[-8.4921, -6.8208, 5.8099, 9.4989]], grad_fn=<SliceBackward>)
直接设置参数
#直接设置参数
net[0].weight.data[:] += 1
net[0].weight.data[0,0] = 42
print(net[0].weight.data[0])
tensor([42.0000, 1.0000, -5.1510, 8.2907])
参数绑定
多个层间共享参数:定义一个稠密层,然后使用它的参数设置另一层的参数
#参数绑定
shared = nn.Linear(8,8)
net = nn.Sequential(nn.Linear(4,8),
nn.ReLU(),
shared,
nn.ReLU(),
shared,
nn.ReLU(),
nn.Linear(8,1))
net(x)
print(net[2].weight.data[0] == net[4].weight.data[0])
net[2].weight.data[0,0] = 100
print(net[2].weight.data[0] == net[4].weight.data[0])
tensor([True, True, True, True, True, True, True, True])
tensor([True, True, True, True, True, True, True, True])
自定义层
不带参数的层
import torch
from torch import nn
import torch.nn.functional as F
class CenteredLayer(nn.Module):
def __init__(self):
super(CenteredLayer, self).__init__()
def forward(self,x):
return x - x.mean()
layer = CenteredLayer()
print(layer(torch.FloatTensor([1,2,3,4,5])))
tensor([-2., -1., 0., 1., 2.])
将其组合到更复杂的模型中’
net = nn.Sequential(nn.Linear(8,128),CenteredLayer())
Y = net(torch.rand(4,8))
print(Y.mean())
tensor(7.4506e-09, grad_fn=<MeanBackward0>)
带参数的层
class MyLinear(nn.Module):
def __init__(self,inputs,outputs):
super(MyLinear, self).__init__()
self.weight = nn.Parameter(torch.randn(inputs,outputs))
self.bias = nn.Parameter(torch.randn(outputs,))
def forward(self,x):
linear = torch.matmul(x,self.weight.data) + self.bias.data
return F.relu(linear)
linear = MyLinear(5,3)
print(linear.weight)
print(linear(torch.rand(2,5)))
Parameter containing:
tensor([[ 0.6429, -0.5441, -0.3412],
[-0.2366, 0.3360, 0.1640],
[ 0.5280, -0.7468, 0.0318],
[ 1.4163, -0.2793, 0.7133],
[-1.6705, -1.8881, 1.2714]], requires_grad=True)
tensor([[0.0000, 0.0000, 2.7141],
[0.0000, 0.0000, 1.4914]])
使用自定义层构建模型
net = nn.Sequential(MyLinear(64,8),MyLinear(8,1))
print(net(torch.rand(2,64)))
tensor([[3.7246],
[3.9040]])
总结:
- 可以通过基本层类设计自定义层
- 在自定义层定义完成后,可以在任意环境和网络构架中调用该自定义层
- 层可以有局部参数,这些参数可以通过内置函数创建
读写文件
对于单个张量,直接调用load和save函数分别读写它们。
import torch
from torch import nn
from torch.nn import functional as F
x = torch.arange(4)
torch.save(x,'x_file')
x2 = torch.load('x_file')
print(x2)
tensor([0, 1, 2, 3])
张量列表的存储与加载
y = torch.ones(4)
torch.save([x,y],'x_files')
x2,y2 = torch.load('x_files')
print((x2,y2))
(tensor([0, 1, 2, 3]), tensor([1., 1., 1., 1.]))
张量字典的存储与加载
mydict = {'x':x,'y':y}
torch.save(mydict,'mydict')
mydict1 = torch.load('mydict')
print(mydict1)
{'x': tensor([0, 1, 2, 3]), 'y': tensor([1., 1., 1., 1.])}
加载和保存模型参数
class MLP(nn.Module):
def __init__(self):
super(MLP, self).__init__()
self.linear1 = nn.Linear(20,256)
self.linear2 = nn.Linear(256,10)
def forward(self,x):
x = self.linear1(x)
x = F.relu(x)
x = self.linear2(x)
return x
net = MLP()
x = torch.randn((2,20))
y = net(x)
torch.save(net.state_dict(),'mlp.params')
clone = MLP()
clone.load_state_dict(torch.load('mlp.params'))
clone.eval()
print(clone)
y_clone = clone(x)
print(y_clone == y)
MLP(
(linear1): Linear(in_features=20, out_features=256, bias=True)
(linear2): Linear(in_features=256, out_features=10, bias=True)
)
tensor([[True, True, True, True, True, True, True, True, True, True],
[True, True, True, True, True, True, True, True, True, True]])
总结:
- save和load函数可用于张量对象的文件读写
- 可以通过参数字典保存和加载网络的全部参数
- 保存框架必须在代码中完成,而不是在参数中完成
GPU
查看显卡信息
!nvidia-smi
查询可用GPU的数量
torch.cuda.device_count()
神经网络与GPU
device=torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model=model.to(device)