4.2.2 激活函数

1.最常用的激活函数ReLU

数学表达式： ReLU(x) = max(0,x)

import torch as t
from torch import nn
from torch.autograd import Variable as V

relu = nn.ReLU(inplace=True) # inplace = True代表它会把输出直接覆盖到输入中，这样可以节省内存/显存‘
input = V(t.randn(2,3))
print(input)
output = relu(input)   # 等价于input.clamp(min=0)
print(output)

2.前馈神经网络简化forward函数的方法

1. 对于前馈传播网络（前一层的输出作为下一层的输入），如果每次都写复杂的forward函数的话会有一些麻烦
2. 有两种简化方法，ModuleList和Sequential。

Sequential代码

import torch as t
from torch import nn
from torch.autograd import Variable as V

# Sequential 三种写法
net1 = nn.Sequential()
net1.add_module('conv',nn.Conv2d(3,3,(3,3)))  # 卷积核个数为3
net1.add_module('batchnorm',nn.BatchNorm2d(3))
net1.add_module('activation_layer',nn.ReLU())

net2 = nn.Sequential(nn.Conv2d(3,3,3),nn.BatchNorm2d(3),nn.ReLU())

from collections import OrderedDict
net3 = nn.Sequential(OrderedDict([
    ('conv1',nn.Conv2d(3,3,3)),
    ('bn1',nn.BatchNorm2d(3)),
    ('relu1',nn.ReLU())
    ]))

print('net1',net1)
print('net2',net2)
print('net3',net3)

#可以根据名字或者序号取出子module
print('......................................................')
print(net1.conv)
print(net2[0])
print(net3.conv1)

print('......................................................')
input = V(t.randn(1,3,4,4))

output = net1(input)
print(output)
output = net2(input)
print(output)
output = net3(input)
print(output)
output = net3.relu1(net1.batchnorm(net1.conv(input)))

输出：

net1 Sequential(
  (conv): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1))
  (batchnorm): BatchNorm2d(3, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (activation_layer): ReLU()
)
net2 Sequential(
  (0): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1))
  (1): BatchNorm2d(3, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (2): ReLU()
)
net3 Sequential(
  (conv1): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1))
  (bn1): BatchNorm2d(3, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (relu1): ReLU()
)
......................................................
Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1))
Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1))
Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1))
......................................................
......................................................
tensor([[[[0.0000, 0.8485],
          [0.0000, 0.6886]],

         [[0.0000, 0.1103],
          [0.2003, 0.5588]],

         [[0.0000, 0.0000],
          [0.0000, 1.5874]]]], grad_fn=<ThresholdBackward0>)
tensor([[[[1.0474, 0.0077],
          [0.0790, 0.0000]],

         [[0.1522, 0.8856],
          [0.0000, 0.0000]],

         [[0.0000, 0.0000],
          [0.1706, 0.1127]]]], grad_fn=<ThresholdBackward0>)
tensor([[[[0.3883, 0.0000],
          [0.0000, 0.0269]],

         [[0.0610, 0.1589],
          [0.0000, 0.0000]],

         [[0.0000, 0.0358],
          [0.0000, 0.0396]]]], grad_fn=<ThresholdBackward0>)

ModuleList代码

import torch as t
from torch import nn
from torch.autograd import Variable as V 

modellist = nn.ModuleList([nn.Linear(3,4),nn.ReLU(),nn.Linear(4,2)])
input = V(t.randn(1,3))
for model in modellist: #modellist没有实现forward，所以不能直接这样写：outout = modellist(input)
    input = model(input)

print(input)

输出：

tensor([[1.2580, 0.8318]], grad_fn=<AddmmBackward>)

3. 测试ModuleList中的子Module可以被主Module识别，而普通的List中的却不能

import torch as t
from torch import nn
from torch.autograd import Variable as V

class MyModule(nn.Module):
    def __init__(self):
        super(MyModule,self).__init__() # 显示调用父类构造函数
        self.list = [nn.Linear(2,3),nn.ReLU()]
        self.module_list = nn.ModuleList([nn.Conv2d(3,3,3),nn.ReLU()])
    def forward(self, *input):
        pass

model = MyModule()
print(model)

for name,param in model.named_parameters():
    print(name,param.size())

输出结果：

MyModule(
  (module_list): ModuleList(
    (0): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1))
    (1): ReLU()
  )
)
module_list.0.weight torch.Size([3, 3, 3, 3])
module_list.0.bias torch.Size([3])

可见，list中的子module不能被主module识别，而ModuleList中的子module能够被module识别。