上一章Googlenet对于我们学习算法的人来说感觉已经比较复杂了,毕竟深度那么大。但是,学无止境,在2015年的时候,又出现了一个神奇的网络——ResNet。
我们知道,神经网络越深,效果越好,但这只能是理论上的,实际上神经网络越深,在误差反传的过程中,梯度(可以认为是信息量)会慢慢减小,甚至为0,这就是的神经网络难以训练,致使效果不好。在ResNet出现以前,我们要解决这种现象有两种途径:
1)按层训练。先训练靠近数据的层,随后慢慢叠加,但这种方式一来非常麻烦,二来效果还是不好;
2)增加宽度而不是深度。比如增加特征图数或全连接神经元数,但还是不如增加深度好使(变宽很容易过拟合)。
ResNet通过增加跨层的连接来解决梯度逐层回传时变小的问题。下图简单的表现了一个跨层连接:
这样做,最底层的输入不仅是给了下一层,且梯度可以沿着跨连接的方式回传到最底层,如下:
为什么叫残差网络呢?我们可以将结构拆开来看:
残差结构可以认为是一个简单结构和复杂结构的和,一层一个,一层两个,最下面的层是共享的。在训练的时候,左边的网络因为更简单所以更容易训练,而这个小网络没有拟合到的信息,被右边的结构获取。关于具体的原理部分需参考残差网络(Deep Residual Learning for Image Recognition)这篇博文,我觉得写的蛮好。
Residual块
ResNet继续使用VGGNet那样的3×3卷积,并且添加了BN层来加速训练,每次跨层连接跨越2个卷积层,这里我们定义这样一个Block:如果输入输出通道数不一致,我们使用1×1卷积,同时设置步长来减小w和h。
Residual定义:
class Residual(gn.nn.Block):
def __init__(self,channels,same_shape=True,**kwargs):
super(Residual, self).__init__(**kwargs)
'''
:param channels: 输出通道数
:param same_shape: 输入和输出的shape是否要一样,因为可能要做加法
'''
if same_shape:
strides=1
else:
strides=2
with self.name_scope():
self.conv1=gn.nn.Conv2D(channels,kernel_size=3,padding=1,strides=strides)
self.bn1=gn.nn.BatchNorm()
self.conv2 = gn.nn.Conv2D(channels, kernel_size=3, padding=1)
self.bn2 = gn.nn.BatchNorm()
if not same_shape: # 如果输出通道数不一致,启用1X1卷积
self.conv3=gn.nn.Conv2D(channels,kernel_size=1,strides=strides)
def forward(self, x):
out=nd.relu(self.bn1(self.conv1(x)))
out=self.bn2(self.conv2(out))
if not self.same_shape: # 如果维度不一样
x=self.conv3(x) # 这样out和x的维度就一样了
return nd.relu(out+x)
我们运行一个实例看看:
1) 当输入输出通道数一样时
# 运行一个实例看看
blk=Residual(channels=3)
blk.initialize()
x=nd.random_normal(shape=(1,3,8,8)) # NCHW
print(blk(x).shape)
结果:
2)当输入输出通道数不一致时
blk=Residual(channels=16,same_shape=False)
blk.initialize()
x=nd.random_normal(shape=(1,3,8,8)) # NCHW
print(blk(x).shape)
运行结果:
下面定义一个残差网络:
'''---ResNet模型定义---'''
def ResNet():
# 随便写一个残差,依旧分为6块
# block 1
b1=gn.nn.Sequential()
b1.add(gn.nn.Conv2D(channels=32,kernel_size=5,strides=2))
# block 2
b2=gn.nn.Sequential()
b2.add(Residual(channels=64,same_shape=False),# 因为上一是32,所以维度不一样
Residual(channels=64))
# block 3
b3 = gn.nn.Sequential()
b3.add(Residual(channels=128, same_shape=False),
Residual(channels=128))
# block 4
b4 = gn.nn.Sequential()
b4.add(Residual(channels=256, same_shape=False),
Residual(channels=256))
# block 5
b5 = gn.nn.Sequential()
b5.add(Residual(channels=512, same_shape=False),
Residual(channels=512))
# block 6
b6 = gn.nn.Sequential()
b6.add(gn.nn.GlobalAvgPool2D(),
gn.nn.Dense(10)) # 10个分类
# 结合所有block
net=gn.nn.Sequential()
net.add(b1,b2,b3,b4,b5,b6)
return net
跑一个实例看看:
net=ResNet()
net.initialize()
X=nd.random_normal(shape=(1,3,32,32))
for layer in net:
X = layer(X)
print(layer.name, 'output shape:\t', X.shape)
结果:
下面放上所有代码:
import mxnet.ndarray as nd
import mxnet.autograd as ag
import mxnet.gluon as gn
import mxnet as mx
import matplotlib.pyplot as plt
from mxnet import init
class Residual(gn.nn.Block):
def __init__(self,channels,same_shape=True,**kwargs):
super(Residual, self).__init__(**kwargs)
'''
:param channels: 输出通道数
:param same_shape: 输入和输出的shape是否要一样,因为可能要做加法
'''
self.same_shape=same_shape
with self.name_scope():
if same_shape:
strides = 1
else:
strides = 2
self.conv1=gn.nn.Conv2D(channels,kernel_size=3,padding=1,strides=strides)
self.bn1=gn.nn.BatchNorm()
self.conv2 = gn.nn.Conv2D(channels, kernel_size=3, padding=1)
self.bn2 = gn.nn.BatchNorm()
if not same_shape: # 如果输出通道数不一致,启用1X1卷积
self.conv3=gn.nn.Conv2D(channels,kernel_size=1,strides=strides)
def forward(self, x):
out=nd.relu(self.bn1(self.conv1(x)))
out=self.bn2(self.conv2(out))
if not self.same_shape: # 如果维度不一样
x=self.conv3(x) # 这样out和x的维度就一样了
return nd.relu(out+x)
# 运行一个实例看看
# blk=Residual(channels=16,same_shape=False)
# blk.initialize()
# x=nd.random_normal(shape=(1,3,8,8)) # NCHW
# print(blk(x).shape)
ctx=mx.gpu()
'''---ResNet模型定义---'''
def ResNet():
# 随便写一个残差,依旧分为6块
# block 1
b1=gn.nn.Sequential()
b1.add(gn.nn.Conv2D(channels=32,kernel_size=5,strides=2))
# block 2
b2=gn.nn.Sequential()
b2.add(Residual(channels=64,same_shape=False),# 因为上一是16,所以维度不一样
Residual(channels=64))
# block 3
b3 = gn.nn.Sequential()
b3.add(Residual(channels=128, same_shape=False),
Residual(channels=128))
# block 4
b4 = gn.nn.Sequential()
b4.add(Residual(channels=256, same_shape=False),
Residual(channels=256))
# block 5
b5 = gn.nn.Sequential()
b5.add(Residual(channels=512, same_shape=False),
Residual(channels=512))
# block 6
b6 = gn.nn.Sequential()
b6.add(gn.nn.GlobalAvgPool2D(),
gn.nn.Dense(10)) # 10个分类
# 结合所有block
net=gn.nn.Sequential()
net.add(b1,b2,b3,b4,b5,b6)
return net
net=ResNet()
net.initialize(init=init.Xavier(),ctx=ctx)
# X=nd.random_normal(shape=(1,3,32,32))
# for layer in net:
# X = layer(X)
# print(layer.name, 'output shape:\t', X.shape)
'''---读取数据和预处理---'''
def load_data_fashion_mnist(batch_size, resize=None):
transformer = []
if resize:
transformer += [gn.data.vision.transforms.Resize(resize)]
transformer += [gn.data.vision.transforms.ToTensor()]
transformer = gn.data.vision.transforms.Compose(transformer)
mnist_train = gn.data.vision.FashionMNIST(train=True)
mnist_test = gn.data.vision.FashionMNIST(train=False)
train_iter = gn.data.DataLoader(
mnist_train.transform_first(transformer), batch_size, shuffle=True)
test_iter = gn.data.DataLoader(
mnist_test.transform_first(transformer), batch_size, shuffle=False)
return train_iter, test_iter
batch_size=128
train_iter,test_iter=load_data_fashion_mnist(batch_size,resize=96) # 96,因为图片加大的话训练很慢,而且显存会吃不消
# 定义准确率
def accuracy(output,label):
return nd.mean(output.argmax(axis=1)==label).asscalar()
def evaluate_accuracy(data_iter,net):# 定义测试集准确率
acc=0
for data,label in data_iter:
data, label = data.as_in_context(ctx), label.as_in_context(ctx)
label = label.astype('float32')
output=net(data)
acc+=accuracy(output,label)
return acc/len(data_iter)
# softmax和交叉熵分开的话数值可能会不稳定
cross_loss=gn.loss.SoftmaxCrossEntropyLoss()
# 优化
train_step=gn.Trainer(net.collect_params(),'sgd',{"learning_rate":0.2}) #因为使用了BN,所以学习率可以大一些
# 训练
lr=0.1
epochs=20
for epoch in range(epochs):
n=0
train_loss=0
train_acc=0
for image,y in train_iter:
image, y = image.as_in_context(ctx), y.as_in_context(ctx)
y = y.astype('float32')
with ag.record():
output = net(image)
loss = cross_loss(output, y)
loss.backward()
train_step.step(batch_size)
train_loss += nd.mean(loss).asscalar()
train_acc += accuracy(output, y)
test_acc = evaluate_accuracy(test_iter, net)
print("Epoch %d, Loss:%f, Train acc:%f, Test acc:%f"
%(epoch,train_loss/len(train_iter),train_acc/len(train_iter),test_acc))
训练结果: