16-参数管理 动手深度学习

#参数管理
import torch
from torch import nn
#具有单隐藏层的多层感知机
net = nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,1))
x = torch.rand(size=(2,4))
print(net(x))
#OrderedDict([('weight', tensor([[ 0.2502,  0.2609,  0.2228, -0.3122, -0.2007,  0.0709, -0.2987,  0.0753]])), ('bias', tensor([0.1400]))])
print(net[2].state_dict())#参数访问 拿到最后一个输出层的参数
print(type(net[2].bias))#<class 'torch.nn.parameter.Parameter'>
print(net[2].bias)#tensor([0.1400], requires_grad=True)
print(net[2].bias.data)#tensor([0.1400])

print(*[(name,param.shape) for name,param in net[0].named_parameters()])
print(*[(name,param.shape) for name,param in net.named_parameters()])

#从嵌套块收集参数
def block1():
    return nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,4),nn.ReLU())
def block2():
    net = nn.Sequential()
    for i in range(4):
        net.add_module(f'block {i}',block1())#block2嵌套4个block1
    return net
rgnet = nn.Sequential(block2(),nn.Linear(4,1))
print(rgnet(x))
print(rgnet)

#内置初始化
def init_normal(m):
    if type(m) == nn.Linear:
        nn.init.normal_(m.weight,mean=0,std=0.01)#对weights做均值为0 标准差为0.01的初始化
        nn.init.zeros_(m.bias)
net.apply(init_normal)
print(net[0].weight.data[0],net[0].bias.data[0])#tensor([0.0112, 0.0118, 0.0126, 0.0207]) tensor(0.)

def init_constant(m):
    if type(m) == nn.Linear:
        nn.init.constant_(m.weight,1)#被初始化为唯一值（只是可以这样做）
        nn.init.zeros_(m.bias)
net.apply(init_constant)
print(net[0].weight.data[0],net[0].bias.data[0])#tensor([1., 1., 1., 1.]) tensor(0.)

#对某些快应用不同的初始化方法
def xavier(m):
    if type(m) == nn.Linear:
        nn.init.xavier_uniform_(m.weight)#xvaier初始化就是实现一种均匀分布
def init_42(m):
    if type(m) == nn.Linear:
        nn.init.constant_(m.weight,42)
net[0].apply(xavier)
net[2].apply(init_42)
print(net[0].weight.data[0])#tensor([ 0.4168,  0.6258, -0.5716,  0.1742])
print(net[2].weight.data)#tensor([[42., 42., 42., 42., 42., 42., 42., 42.]])

#自定义初始化
def my_init(m):
    if type(m) == nn.Linear:
        print(
            "Init",
            *[(name,param.shape) for name,param in m.named_parameters()][0])
        nn.init.uniform_(m.weight,-10,10)
        m.weight.data *= m.weight.data.abs() >= 5#保留绝对值大于等于5的权重
net.apply(my_init)
print(net[0].weight[:2])