paddle 图像分割学习总结

提示：文章写完后，目录可以自动生成，如何生成可参考右边的帮助文档

---

前言

课程链接：https://aistudio.baidu.com/aistudio/course/introduce/1767

---

提示：以下是本篇文章正文内容

一、图像分割综述

根据 不同的任务和数据类型：

-图像分割（image segmentation）（像素级分类）

• 图像语义分割（image semantic segmentation）
• 图像实例分割（image instance segmentation）
• 图像全景分割（image panoptic segmentation）
• 视频目标分割（video object segmentation）
• 视频实例分割（video instance segmentation）

---

示例：

• 语义分割：给每个pixel分类
• 实例分割：给每个筐里的object分mask
• 全景分割：背景pixel分类+框里mask
• VOS ：通常会给定目标的mask，求特定目标的mask
• VIS ：根据目标检测的框，求目标的mask
  

---

分割网络的评价指标：mIoU 和 mAcc

• mean intersection-over-unio
  分割每一类别的交并比
• mean accuracy
  分对的像素 / 所有像素

二、FCN

网络结构：

代码如下（示例）：

import numpy as np
import paddle.fluid as fluid
from paddle.fluid.dygraph import to_variable
from paddle.fluid.dygraph import Conv2D
from paddle.fluid.dygraph import Conv2DTranspose
from paddle.fluid.dygraph import Dropout
from paddle.fluid.dygraph import BatchNorm
from paddle.fluid.dygraph import Pool2D
from paddle.fluid.dygraph import Linear
# from vgg import VGG16BN

class VGG(fluid.dygraph.Layer):
    def __init__(self, layers=16, use_bn=False, num_classes=1000):
        super(VGG, self).__init__()
        self.layers = layers
        self.use_bn = use_bn
        supported_layers = [16, 19]
        assert layers in supported_layers

        if layers == 16:
            depth = [2, 2, 3, 3, 3]
        elif layers == 19:
            depth = [2, 2, 4, 4, 4]

        num_channels = [3, 64, 128, 256, 512]
        num_filters = [64, 128, 256, 512, 512]

        self.layer1 = fluid.dygraph.Sequential(*self.make_layer(num_channels[0], num_filters[0], depth[0], use_bn, name='layer1'))
        self.layer2 = fluid.dygraph.Sequential(*self.make_layer(num_channels[1], num_filters[1], depth[1], use_bn, name='layer2'))
        self.layer3 = fluid.dygraph.Sequential(*self.make_layer(num_channels[2], num_filters[2], depth[2], use_bn, name='layer3'))
        self.layer4 = fluid.dygraph.Sequential(*self.make_layer(num_channels[3], num_filters[3], depth[3], use_bn, name='layer4'))
        self.layer5 = fluid.dygraph.Sequential(*self.make_layer(num_channels[4], num_filters[4], depth[4], use_bn, name='layer5'))

        self.classifier = fluid.dygraph.Sequential(
                Linear(input_dim=512 * 7 * 7, output_dim=4096, act='relu'),
                Dropout(),
                Linear(input_dim=4096, output_dim=4096, act='relu'),
                Dropout(),
                Linear(input_dim=4096, output_dim=num_classes))
                
        self.out_dim = 512 * 7 * 7


    def forward(self, inputs):
        x = self.layer1(inputs)
        x = fluid.layers.pool2d(x, pool_size=2, pool_stride=2)
        x = self.layer2(x)
        x = fluid.layers.pool2d(x, pool_size=2, pool_stride=2)
        x = self.layer3(x)
        x = fluid.layers.pool2d(x, pool_size=2, pool_stride=2)
        x = self.layer4(x)
        x = fluid.layers.pool2d(x, pool_size=2, pool_stride=2)
        x = self.layer5(x)
        x = fluid.layers.pool2d(x, pool_size=2, pool_stride=2)
        x = fluid.layers.adaptive_pool2d(x, pool_size=(7,7), pool_type='avg')
        x = fluid.layers.reshape(x, shape=[-1, self.out_dim])
        x = self.classifier(x)

        return x

class FCN8s(fluid.dygraph.Layer):
 # TODO: create fcn8s model
    def __init__(self, num_classes=59):
        super(FCN8s, self).__init__()
        self.num_classes = num_classes
        
        self.layer1 = fluid.dygraph.Sequential(
            Conv2D(num_channels=3, num_filters=64, filter_size=3, padding=1),
            BatchNorm(num_channels=64, act='relu'),
            Conv2D(num_channels=64, num_filters=64, filter_size=3, padding=1),
            BatchNorm(num_channels=64, act='relu'),
            Pool2D(pool_size=2, pool_stride=2, pool_type='max')
        ) # 1/ 2
        self.layer2 = fluid.dygraph.Sequential(
            Conv2D(num_channels=64, num_filters=128, filter_size=3, padding=1),
            BatchNorm(num_channels=128, act='relu'),
            Conv2D(num_channels=128, num_filters=128, filter_size=3, padding=1),
            BatchNorm(num_channels=128, act='relu'),
            Pool2D(pool_size=2, pool_stride=2, pool_type='max')
        ) # 1/ 4
        self.layer3 = fluid.dygraph.Sequential(
            Conv2D(num_channels=128, num_filters=256, filter_size=3, padding=1),
            BatchNorm(num_channels=256, act='relu'),
            Conv2D(num_channels=256, num_filters=256, filter_size=3, padding=1),
            BatchNorm(num_channels=256, act='relu'),
            Conv2D(num_channels=256, num_filters=256, filter_size=3, padding=1),
            BatchNorm(num_channels=256, act='relu'),
            Pool2D(pool_size=2, pool_stride=2, pool_type='max')
        ) # 1 / 8
        self.layer4 = fluid.dygraph.Sequential(
            Conv2D(num_channels=256, num_filters=512, filter_size=3, padding=1),
            BatchNorm(num_channels=512, act='relu'),
            Conv2D(num_channels=512, num_filters=512, filter_size=3, padding=1),
            BatchNorm(num_channels=512, act='relu'),
            Conv2D(num_channels=512, num_filters=512, filter_size=3, padding=1),
            BatchNorm(num_channels=512, act='relu'),
            Pool2D(pool_size=2, pool_stride=2, pool_type='max')
        ) # 1 / 16
        self.layer5 = fluid.dygraph.Sequential(
            Conv2D(num_channels=512, num_filters=512, filter_size=3, padding=1),
            BatchNorm(num_channels=512, act='relu'),
            Conv2D(num_channels=512, num_filters=512, filter_size=3, padding=1),
            BatchNorm(num_channels=512, act='relu'),
            Conv2D(num_channels=512, num_filters=512, filter_size=3, padding=1),
            BatchNorm(num_channels=512, act='relu'),
            Pool2D(pool_size=2, pool_stride=2, pool_type='max')
        ) # 1 / 32
        self.conv67 = fluid.dygraph.Sequential(
            Conv2D(num_channels=512, num_filters=512, filter_size=1),
            BatchNorm(num_channels=512, act='relu'),
            Conv2D(num_channels=512, num_filters=512, filter_size=1),
            BatchNorm(num_channels=512, act='relu')
        )
        self.conv = Conv2D(num_channels=512, num_filters=self.num_classes, filter_size=1)
    
    def forward(self, inputs):
        x = self.layer1(inputs)
        x = self.layer2(x)
        x = self.layer3(x)
        pool3 = x # 1/8 256
        x = self.layer4(x)
        pool4 = x # 1/16 512
        x = self.layer5(x)
        x = self.conv67(x)

        x = fluid.layers.interpolate(x, pool4.shape[2:])
        x = fluid.layers.elementwise_add(pool4, x)
        x = fluid.layers.interpolate(x, pool3.shape[2:])
        pool3 = Conv2D(num_channels= 256, num_filters=512, filter_size=1, act='relu')(pool3)
        x = fluid.layers.elementwise_add(pool3, x)
        x = fluid.layers.interpolate(x, inputs.shape[2:])
        x = self.conv(x)

        return x

def main():
    with fluid.dygraph.guard():
        x_data = np.random.rand(2, 3, 512, 512).astype(np.float32)
        x = to_variable(x_data)
        model = FCN8s(num_classes=59)
        model.eval()
        pred = model(x)
        print(pred.shape)


if __name__ == '__main__':
    main()

U-Net

网络结构：

• 采用编码器和解码器的U形结构
• 输入输出大小不变
• skip结合方式：concatenation
  

Paper Title: U-net:Convolutional networks for biomedical image segmentaion

PSPNet

psp模块

import numpy as np
import paddle.fluid as fluid
from paddle.fluid.dygraph import to_variable
from paddle.fluid.dygraph import Layer
from paddle.fluid.dygraph import Conv2D
from paddle.fluid.dygraph import BatchNorm
from paddle.fluid.dygraph import Dropout
from resnet_dilated import ResNet50

# pool with different bin_size
# interpolate back to input size
# concat
class PSPModule(Layer):
    def __init__(self, num_channels, bin_size_list):
        super(PSPModule, self).__init__()
        self.bin_size_list = bin_size_list
        num_filters = num_channels // len(bin_size_list)
        self.features = []
        for i in range(len(bin_size_list)):
            self.features.append(
                fluid.dygraph.Sequential(
                    Conv2D(num_channels, num_filters, 1),
                    BatchNorm(num_filters, act='relu')
                )
            )
    def forward(self, inputs):
        out = [inputs]
        for idx, f in enumerate(self.features):
            x = fluid.layers.adaptive_pool2d(inputs, self.bin_size_list[idx])
            x = f(x)
            x = fluid.layers.interpolate(x, inputs.shape[2:], align_corners=True)
            out.append(x)
        out = fluid.layers.concat(out, axis=1)

        return out


class PSPNet(Layer):
    def __init__(self, num_classes=59, backbone='resnet50'):
        super(PSPNet, self).__init__()
        res = ResNet50()
        # stem: res.conv, res.pool2d_max
        self.layer0 = fluid.dygraph.Sequential(
            res.conv,
            res.pool2d_max
        )
        self.layer1 = res.layer1
        self.layer2 = res.layer2
        self.layer3 = res.layer3
        self.layer4 = res.layer4
        num_channels = 2048
        # psp: 2048 -> 2048*2
        self.pspmodule = PSPModule(num_channels, [1,2,3,6])
        num_channels *= 2
        # cls: 2048*2 -> 512 -> num_classes
        self.classifier = fluid.dygraph.Sequential(
            Conv2D(num_channels = num_channels, num_filters=512, filter_size=3, padding=1),
            BatchNorm(512, act = 'relu'),
            Dropout(0.1),
            Conv2D(num_channels=512, num_filters=num_classes, filter_size=1)
        )
        # aux: 1024 -> 256 -> num_classes
        
    def forward(self, inputs):

        # aux: tmp_x = layer3
        x = self.layer0(inputs)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        aux = x
        x = self.layer4(x)
        x = self.pspmodule(x)
        x = self.classifier(x)
        x = fluid.layers.interpolate(x, inputs.shape[2:])
        return x, aux

def main():
    with fluid.dygraph.guard(fluid.CPUPlace()):
        x_data=np.random.rand(2,3, 473, 473).astype(np.float32)
        x = to_variable(x_data)
        model = PSPNet(num_classes=59)
        model.train()
        pred, aux = model(x)
        print(pred.shape, aux.shape)

if __name__ =="__main__":
    main()

DeepLab

aspp模块升级版

class ASPPPooling(Layer):
    # TODO:
    def __init__(self, num_channels, num_filters):
        super(ASPPPooling, self).__init__()
        self.features = fluid.dygraph.Sequential(
            Conv2D(num_channels, num_filters, 1),
            BatchNorm(num_filters, act='relu')
        )
    def forward(self, inputs):
        n, c, h, w = inputs.shape

        x = fluid.layers.adaptive_pool2d(inputs, 1)
        x = self.features(x)
        x = fluid.layers.interpolate(x, [h, w])

        return x


class ASPPConv(fluid.dygraph.Sequential):
    # TODO:
    def __init__(self, num_channels, num_filters, dilation):
        super(ASPPConv, self).__init__(
            Conv2D(num_channels=num_channels, num_filters=num_filters, filter_size=3, padding=dilation, dilation=dilation),
            BatchNorm(num_filters, act='relu')
        )




class ASPPModule(Layer):
    # TODO: 
    def __init__(self, num_channels, num_filters, rates):
        super(ASPPModule, self).__init__()
        self.features = []
        self.features.append(
            fluid.dygraph.Sequential(
                Conv2D(num_channels, num_filters, 1),
                BatchNorm(num_filters, act='relu')
            )
        )
        self.features.append(ASPPPooling(num_channels, num_filters))    
        for r in rates:
            self.features.append(ASPPConv(num_channels, num_filters, r))

        self.project = fluid.dygraph.Sequential(
            Conv2D(1280, 256, 1),
            BatchNorm(256, act='relu')
        )

    def forward(self, inputs):
        res = []
        for f in self.features:
            res.append(f(inputs))
        x = fluid.layers.concat(res, axis=1)
        x = self.project(x)

        return x


class DeepLabHead(fluid.dygraph.Sequential):
    def __init__(self, num_channels, num_classes):
        super(DeepLabHead, self).__init__(
                ASPPModule(num_channels, 256, [12, 24, 36]),
                Conv2D(256, 256, 3, padding=1),
                BatchNorm(256, act='relu'),
                Conv2D(256, num_classes, 1)
                )


class DeepLab(Layer):
    # TODO:
    def __init__(self, num_classes=59):
        super(DeepLab, self).__init__()
        bone = ResNet50(pretrained=False, duplicate_blocks=True)
        self.layer0 = fluid.dygraph.Sequential(
            bone.conv,
            bone.pool2d_max
        )
        self.layer1 = bone.layer1
        self.layer2 = bone.layer2
        self.layer3 = bone.layer3
        self.layer4 = bone.layer4
        #multi grid
        self.layer5 = bone.layer5
        self.layer6 = bone.layer6
        self.layer7 = bone.layer7
        # feature_dim = 2048
        self.classifier = DeepLabHead(2048, 59)

    def forward(self, inputs):
        n, c, h, w = inputs.shape
        x = self.layer0(inputs)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.layer5(x)
        x = self.layer6(x)
        x = self.layer7(x)
        x = self.classifier(x)
        x = fluid.layers.interpolate(x, [h, w], align_corners=False)

        return x