DeepLab V3
Rethinking Atrous Convolution for Semantic Image Segmentation
论文地址:https://arxiv.org/abs/1706.05587下文中图片来自飞桨图像分割7日训练营课程ppt
(指路)https://aistudio.baidu.com/aistudio/course/introduce/1767
代码部分
[1] 实现(debug后)
# deeplabv3.py
import numpy as np
import paddle.fluid as fluid
from paddle.fluid.dygraph import to_variable
from paddle.fluid.dygraph import Layer
from paddle.fluid.dygraph import Conv2D
from paddle.fluid.dygraph import BatchNorm
from paddle.fluid.dygraph import Dropout
from resnet_multi_grid import ResNet50, ResNet101, ResNet152
class ASPPPooling(Layer):
# TODO:
def __init__(self, num_channels, num_filters):
super(ASPPPooling, self).__init__()
self.features = fluid.dygraph.Sequential(
Conv2D(num_channels, num_filters, 1),
BatchNorm(num_filters, act='relu')
)
def forward(self, inputs):
n, c, h, w = inputs.shape
x = fluid.layers.adaptive_pool2d(inputs, 1)
x = self.features(x)
x = fluid.layers.interpolate(x, (h, w), align_corners=False)
return x
class ASPPConv(fluid.dygraph.Sequential):
# TODO:
def __init__(self, num_channels, num_filters, dilation):
#super(ASPPConv, self).__init__().__init__(
super(ASPPConv, self).__init__(
Conv2D(num_channels, num_filters, filter_size=3,
padding=dilation, dilation=dilation),
BatchNorm(num_filters, act='relu')
)
class ASPPModule(Layer):
# TODO:
def __init__(self, num_channels, num_filters, rates):
super(ASPPModule, self).__init__()
self.features = []
self.features.append(
fluid.dygraph.Sequential(
Conv2D(num_channels, num_filters, 1),
BatchNorm(num_filters, act='relu')
)
)
self.features.append(ASPPPooling(num_channels, num_filters))
for r in rates:
print(r)
self.features.append(
ASPPConv(num_channels, num_filters, r)
)
self.project = fluid.dygraph.Sequential(
Conv2D(num_filters*(2 + len(rates)), num_filters, 1),
BatchNorm(num_filters, act='relu')
)
def forward(self, inputs):
res = []
for op in self.features:
print(op)
res.append(op(inputs))
x = fluid.layers.concat(res, axis=1)
x = self.project(x)
return x
class DeepLabHead(fluid.dygraph.Sequential):
def __init__(self, num_channels, num_classes):
super(DeepLabHead, self).__init__(
ASPPModule(num_channels, 256, [12, 24, 36]),
Conv2D(256, 256, 3, padding=1),
BatchNorm(256, act='relu'),
Conv2D(256, num_classes, 1)
)
class DeepLab(Layer):
# TODO:
def __init__(self, num_classes=59):
super(DeepLab, self).__init__()
resnet = ResNet50(pretrained=False, duplicate_blocks=True)
self.layer0 = fluid.dygraph.Sequential(
resnet.conv,
resnet.pool2d_max
)
self.layer1 = resnet.layer1
self.layer2 = resnet.layer2
self.layer3 = resnet.layer3
self.layer4 = resnet.layer4
# multigrid
self.layer5 = resnet.layer5
self.layer6 = resnet.layer6
self.layer7 = resnet.layer7
feature_dim = 2048
self.classifier = DeepLabHead(feature_dim, num_classes)
def forward(self, inputs):
n, c, h, w = inputs.shape
x = self.layer0(inputs)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = self.layer5(x)
x = self.layer6(x)
x = self.layer7(x)
x = self.classifier(x)
x = fluid.layers.interpolate(x, (h, w), align_corners=False)
return x
def main():
with fluid.dygraph.guard(fluid.CPUPlace()):
x_data = np.random.rand(2, 3, 512, 512).astype(np.float32)
x = to_variable(x_data)
model = DeepLab(num_classes=59)
model.eval()
pred = model(x)
print(pred.shape)
if __name__ == '__main__':
main()
# resnet_multi_grid.py
import numpy as np
import paddle.fluid as fluid
from paddle.fluid.dygraph import to_variable
from paddle.fluid.dygraph import Conv2D
from paddle.fluid.dygraph import BatchNorm
from paddle.fluid.dygraph import Pool2D
from paddle.fluid.dygraph import Linear
model_path = {'ResNet18': './resnet18',
'ResNet34': './resnet34',
'ResNet50': './resnet50',
'ResNet101': './resnet101',
'ResNet152': './resnet152'}
class ConvBNLayer(fluid.dygraph.Layer):
def __init__(self,
num_channels,
num_filters,
filter_size,
stride=1,
groups=1,
act=None,
dilation=1,
padding=None,
name=None):
super(ConvBNLayer, self).__init__(name)
if padding is None:
padding = (filter_size-1)//2
else:
padding=padding
self.conv = Conv2D(num_channels=num_channels,
num_filters=num_filters,
filter_size=filter_size,
stride=stride,
padding=padding,
groups=groups,
act=None,
dilation=dilation,
bias_attr=False)
self.bn = BatchNorm(num_filters, act=act)
def forward(self, inputs):
y = self.conv(inputs)
y = self.bn(y)
return y
class BasicBlock(fluid.dygraph.Layer):
expansion = 1 # expand ratio for last conv output channel in each block
def __init__(self,
num_channels,
num_filters,
stride=1,
shortcut=True,
name=None):
super(BasicBlock, self).__init__(name)
self.conv0 = ConvBNLayer(num_channels=num_channels,
num_filters=num_filters,
filter_size=3,
stride=stride,
act='relu',
name=name)
self.conv1 = ConvBNLayer(num_channels=num_filters,
num_filters=num_filters,
filter_size=3,
act=None,
name=name)
if not shortcut:
self.short = ConvBNLayer(num_channels=num_channels,
num_filters=num_filters,
filter_size=1,
stride=stride,
act=None,
name=name)
self.shortcut = shortcut
def forward(self, inputs):
conv0 = self.conv0(inputs)
conv1 = self.conv1(conv0)
if self.shortcut:
short = inputs
else:
short = self.short(inputs)
y = fluid.layers.elementwise_add(x=short, y=conv1, act='relu')
return y
class BottleneckBlock(fluid.dygraph.Layer):
expansion = 4
def __init__(self,
num_channels,
num_filters,
stride=1,
shortcut=True,
dilation=1,
padding=None,
name=None):
super(BottleneckBlock, self).__init__(name)
self.conv0 = ConvBNLayer(num_channels=num_channels,
num_filters=num_filters,
filter_size=1,
act='relu')
# name=name)
self.conv1 = ConvBNLayer(num_channels=num_filters,
num_filters=num_filters,
filter_size=3,
stride=stride,
padding=padding,
act='relu',
dilation=dilation)
# name=name)
self.conv2 = ConvBNLayer(num_channels=num_filters,
num_filters=num_filters * 4,
filter_size=1,
stride=1)
# name=name)
if not shortcut:
self.short = ConvBNLayer(num_channels=num_channels,
num_filters=num_filters * 4,
filter_size=1,
stride=stride)
# name=name)
self.shortcut = shortcut
self.num_channel_out = num_filters * 4
def forward(self, inputs):
conv0 = self.conv0(inputs)
#print('conv0 shape=',conv0.shape)
conv1 = self.conv1(conv0)
#print('conv1 shape=', conv1.shape)
conv2 = self.conv2(conv1)
#print('conv2 shape=', conv2.shape)
if self.shortcut:
short = inputs
else:
short = self.short(inputs)
#print('short shape=', short.shape)
y = fluid.layers.elementwise_add(x=short, y=conv2, act='relu')
return y
class ResNet(fluid.dygraph.Layer):
def __init__(self, layers=50, num_classes=1000, multi_grid=[1, 2, 4], duplicate_blocks=False):
super(ResNet, self).__init__()
self.layers = layers
supported_layers = [18, 34, 50, 101, 152]
assert layers in supported_layers
mgr = [1, 2, 4] # multi grid rate for duplicated blocks
if layers == 18:
depth = [2, 2, 2, 2]
elif layers == 34:
depth = [3, 4, 6, 3]
elif layers == 50:
depth = [3, 4, 6, 3]
elif layers == 101:
depth = [3, 4, 23, 3]
elif layers == 152:
depth = [3, 8, 36, 3]
if layers < 50:
num_channels = [64, 64, 128, 256, 512]
else:
num_channels = [64, 256, 512, 1024, 2048]
num_filters = [64, 128, 256, 512]
self.conv = ConvBNLayer(num_channels=3,
num_filters=64,
filter_size=7,
stride=2,
act='relu')
self.pool2d_max = Pool2D(pool_size=3,
pool_stride=2,
pool_padding=1,
pool_type='max')
if layers < 50:
block = BasicBlock
l1_shortcut=True
else:
block = BottleneckBlock
l1_shortcut=False
self.layer1 = fluid.dygraph.Sequential(
*self.make_layer(block,
num_channels[0],
num_filters[0],
depth[0],
stride=1,
shortcut=l1_shortcut,
name='layer1'))
self.layer2 = fluid.dygraph.Sequential(
*self.make_layer(block,
num_channels[1],
num_filters[1],
depth[1],
stride=2,
name='layer2'))
self.layer3 = fluid.dygraph.Sequential(
*self.make_layer(block,
num_channels[2],
num_filters[2],
depth[2],
stride=1,
dilation=2,
name='layer3'))
# add multi grid [1, 2, 4]
self.layer4 = fluid.dygraph.Sequential(
*self.make_layer(block,
num_channels[3],
num_filters[3],
depth[3],
stride=1,
name='layer4',
dilation=multi_grid))
if duplicate_blocks:
self.layer5 = fluid.dygraph.Sequential(
*self.make_layer(block,
num_channels[4],
num_filters[3],
depth[3],
stride=1,
name='layer5',
dilation=[x*mgr[0] for x in multi_grid]))
self.layer6 = fluid.dygraph.Sequential(
*self.make_layer(block,
num_channels[4],
num_filters[3],
depth[3],
stride=1,
name='layer6',
dilation=[x*mgr[1] for x in multi_grid]))
self.layer7 = fluid.dygraph.Sequential(
*self.make_layer(block,
num_channels[4],
num_filters[3],
depth[3],
stride=1,
name='layer7',
dilation=[x*mgr[2] for x in multi_grid]))
self.last_pool = Pool2D(pool_size=7, # ignore if global_pooling is True
global_pooling=True,
pool_type='avg')
self.fc = Linear(input_dim=num_filters[-1] * block.expansion,
output_dim=num_classes,
act=None)
self.out_dim = num_filters[-1] * block.expansion
def forward(self, inputs):
x = self.conv(inputs)
x = self.pool2d_max(x)
#print(x.shape)
x = self.layer1(x)
#print(x.shape)
x = self.layer2(x)
#print(x.shape)
x = self.layer3(x)
#print(x.shape)
x = self.layer4(x)
#print(x.shape)
x = self.last_pool(x)
x = fluid.layers.reshape(x, shape=[-1, self.out_dim])
x = self.fc(x)
return x
def make_layer(self, block, num_channels, num_filters, depth, stride, dilation=1, shortcut=False, name=None):
layers = []
if isinstance(dilation, int):
dilation = [dilation] * depth
elif isinstance(dilation, (list, tuple)):
assert len(dilation) == 3, "Wrong dilation rate for multi-grid | len should be 3"
assert depth ==3, "multi-grid can only applied to blocks with depth 3"
padding = []
for di in dilation:
if di>1:
padding.append(di)
else:
padding.append(None)
layers.append(block(num_channels,
num_filters,
stride=stride,
shortcut=shortcut,
dilation=dilation[0],
padding=padding[0],
name=f'{name}.0'))
for i in range(1, depth):
layers.append(block(num_filters * block.expansion,
num_filters,
stride=1,
dilation=dilation[i],
padding=padding[i],
name=f'{name}.{i}'))
return layers
def ResNet18(pretrained=False):
model = ResNet(layers=18)
if pretrained:
model_state, _ = fluid.load_dygraph(model_path['ResNet18'])
model.set_dict(model_state)
return model
def ResNet34(pretrained=False):
model = ResNet(layers=34)
if pretrained:
model_state, _ = fluid.load_dygraph(model_path['ResNet34'])
model.set_dict(model_state)
return model
def ResNet50(pretrained=False, duplicate_blocks=True):
model = ResNet(layers=50, duplicate_blocks=duplicate_blocks)
if pretrained:
model_state, _ = fluid.load_dygraph(model_path['ResNet50'])
if duplicate_blocks:
set_dict_ignore_duplicates(model, model_state)
else:
model.set_dict(model_state)
return model
def findParams(model_state, name):
new_dict = dict()
for key,val in model_state.items():
if name == key[0:len(name)]:
print(f'change {key} -> {key[len(name)+1::]}')
new_dict[key[len(name)+1::]] = val
return new_dict
def set_dict_ignore_duplicates(model, model_state):
model.conv.set_dict(findParams(model_state,'conv'))
model.pool2d_max.set_dict(findParams(model_state,'pool2d_max'))
model.layer1.set_dict(findParams(model_state,'layer1'))
model.layer2.set_dict(findParams(model_state,'layer2'))
model.layer3.set_dict(findParams(model_state,'layer3'))
model.layer4.set_dict(findParams(model_state,'layer4'))
model.fc.set_dict(findParams(model_state,'fc'))
return model
def ResNet101(pretrained=False, duplicate_blocks=False):
model = ResNet(layers=101, duplicate_blocks=duplicate_blocks)
if pretrained:
model_state, _ = fluid.load_dygraph(model_path['ResNet101'])
if duplicate_blocks:
set_dict_ignore_duplicates(model, model_state)
else:
model.set_dict(model_state)
return model
def ResNet152(pretrained=False):
model = ResNet(layers=152)
if pretrained:
model_state, _ = fluid.load_dygraph(model_path['ResNet152'])
model.set_dict(model_state)
return model
def main():
with fluid.dygraph.guard():
#x_data = np.random.rand(2, 3, 512, 512).astype(np.float32)
x_data = np.random.rand(2, 3, 224, 224).astype(np.float32)
x = to_variable(x_data)
#model = ResNet18()
#model.eval()
#pred = model(x)
#print('resnet18: pred.shape = ', pred.shape)
#model = ResNet34()
#pred = model(x)
#model.eval()
#print('resnet34: pred.shape = ', pred.shape)
model = ResNet101(pretrained=False)
model.eval()
pred = model(x)
print('dilated resnet50: pred.shape = ', pred.shape)
#model = ResNet101()
#pred = model(x)
#model.eval()
#print('resnet101: pred.shape = ', pred.shape)
#model = ResNet152()
#pred = model(x)
#model.eval()
#print('resnet152: pred.shape = ', pred.shape)
#print(model.sublayers())
#for name, sub in model.named_sublayers(include_sublayers=True):
# #print(sub.full_name())
# if (len(sub.named_sublayers()))
# print(name)
if __name__ == "__main__":
main()
[2] debug过程
运行后显示通道数出了问题
查看报错,应该是ASPPModule类中,x = self.project(x)处有问题。于是打印forward函数中各变量(或其shape)
可见,inputs没有问题,但是经过concat之后,通道数变为6656,实际应为1280。
则推理为concat中参数res出了问题。而res储存了经过features函数对inputs的操作变换之后的值,可知,是features中的函数出了问题。
回顾ASPPModule类中features的构成,首先添加了一个conv+BN的Sequential,然后是ASPPPooling和三个不同dilation的ASPPConv。那么去看ASPPPooling和ASPPConv两个类,发现在ASPPConv类里,多写了一个__init__(),大概是自动补全的时候敲快了,致使后来输出的通道数剧增。这样写的机制还需研究
改过之后输出目标
成功撒花~
最后,感谢百度飞桨提供的免费平台与课程,朱老师和伍老师讲的都很棒,真的是学到了很多~尤其是朱老师带着敲代码,感觉自己的coding技巧都提升了好多好多倍~~~
有兴趣的小伙伴可以来这里学习哦~
(二度链接)https://aistudio.baidu.com/aistudio/course/introduce/1767
PS:理论部分待补全
参考 https://delphifan.blog.csdn.net/article/details/79144773