PaddleDetection-MaskRcnn结构拆分（二）

2021SC@SDUSC

本文章为PaddleDetection-MaskRcnn的结构拆分

首先是Head部分，算法结构图：

首先是在yaml的配置文件：/configs/_base_/models/mask_rcnn_r50_fpn.yml

'''
RPNHead: #RPNHead类名
  rpn_feat: #rpn特征
    name: RPNFeat #类名
    feat_in: 256 #输入通道数
    feat_out: 256 #输出通道数
  anchor_per_position: 3 #anchor个数
  rpn_channel: 256 #rpn通道数
'''

相关引用库如下：

import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from paddle import ParamAttr
from paddle.nn.initializer import Normal
from paddle.regularizer import L2Decay
from paddle.nn import Conv2D

from ppdet.core.workspace import register
from ppdet.modeling import ops

 候选框网络特征类：

@register
class RPNFeat(nn.Layer):
    def __init__(self, feat_in=1024, feat_out=1024):
        super(RPNFeat, self).__init__()
        # rpn feat is shared with each level
        self.rpn_conv = Conv2D(
            in_channels=feat_in,
            out_channels=feat_out,
            kernel_size=3,
            padding=1,
            weight_attr=ParamAttr(initializer=Normal(
                mean=0., std=0.01)),
            bias_attr=ParamAttr(
                learning_rate=2., regularizer=L2Decay(0.)))

    def forward(self, inputs, feats):
        rpn_feats = []
        for feat in feats:
            rpn_feats.append(F.relu(self.rpn_conv(feat)))
        return rpn_feats

候选框头 

@register
class RPNHead(nn.Layer):
    __inject__ = ['rpn_feat']

    def __init__(self, rpn_feat, anchor_per_position=15, rpn_channel=1024):
        super(RPNHead, self).__init__()
        self.rpn_feat = rpn_feat
        if isinstance(rpn_feat, dict):
            self.rpn_feat = RPNFeat(**rpn_feat)
        # rpn head is shared with each level
        # rpn roi classification scores
        self.rpn_rois_score = Conv2D(
            in_channels=rpn_channel,
            out_channels=anchor_per_position,
            kernel_size=1,
            padding=0,
            weight_attr=ParamAttr(initializer=Normal(
                mean=0., std=0.01)),
            bias_attr=ParamAttr(
                learning_rate=2., regularizer=L2Decay(0.)))

        # rpn roi bbox regression deltas
        self.rpn_rois_delta = Conv2D(
            in_channels=rpn_channel,
            out_channels=4 * anchor_per_position,
            kernel_size=1,
            padding=0,
            weight_attr=ParamAttr(initializer=Normal(
                mean=0., std=0.01)),
            bias_attr=ParamAttr(
                learning_rate=2., regularizer=L2Decay(0.)))

    def forward(self, inputs, feats):
        rpn_feats = self.rpn_feat(inputs, feats)
        rpn_head_out = []
        for rpn_feat in rpn_feats:
            rrs = self.rpn_rois_score(rpn_feat)
            rrd = self.rpn_rois_delta(rpn_feat)
            rpn_head_out.append((rrs, rrd))
        return rpn_feats, rpn_head_out
    #计算RPN loss
    def get_loss(self, loss_inputs):
        # cls loss
        score_tgt = paddle.cast(
            x=loss_inputs['rpn_score_target'], dtype='float32')
        score_tgt.stop_gradient = True
        loss_rpn_cls = ops.sigmoid_cross_entropy_with_logits(
            input=loss_inputs['rpn_score_pred'], label=score_tgt)
        loss_rpn_cls = paddle.mean(loss_rpn_cls, name='loss_rpn_cls')

        # reg loss
        loc_tgt = paddle.cast(x=loss_inputs['rpn_rois_target'], dtype='float32')
        loc_tgt.stop_gradient = True
        loss_rpn_reg = ops.smooth_l1(
            input=loss_inputs['rpn_rois_pred'],
            label=loc_tgt,
            inside_weight=loss_inputs['rpn_rois_weight'],
            outside_weight=loss_inputs['rpn_rois_weight'],
            sigma=3.0, )
        loss_rpn_reg = paddle.sum(loss_rpn_reg)
        score_shape = paddle.shape(score_tgt)
        score_shape = paddle.cast(score_shape, dtype='float32')
        norm = paddle.prod(score_shape)
        norm.stop_gradient = True
        loss_rpn_reg = loss_rpn_reg / norm

        return {'loss_rpn_cls': loss_rpn_cls, 'loss_rpn_reg': loss_rpn_reg}

定义了初始化方法，还有前向传播函数的实现，以及获得损失的方法。

modeling/bbox.py源码解析：

配置文件解析：

'''
Anchor:#锚框
  anchor_generator: #产生锚框
    name: AnchorGeneratorRPN #类名
    aspect_ratios: [0.5, 1.0, 2.0] #锚框比例
    anchor_start_size: 32 #产生锚框尺寸大小
    stride: [4., 4.] #步长
  anchor_target_generator: #目标框
    name: AnchorTargetGeneratorRPN #产生目标框类名
    batch_size_per_im: 256 #输入图片尺寸
    fg_fraction: 0.5 #RPN阈值
    negative_overlap: 0.3 #负例iou置信度
    positive_overlap: 0.7 #正例iou置信度
    straddle_thresh: 0.0  #目标框阈值
Proposal: #目标框
  proposal_generator: #生成目标框
    name: ProposalGenerator #生成目标框类名
    min_size: 0.0 #最小尺寸
    nms_thresh: 0.7 #nms阈值
    train_pre_nms_top_n: 2000 #训练时产生最多框数
    train_post_nms_top_n: 2000 #训练时经后处理后产生最多框数
    infer_pre_nms_top_n: 1000 #推理时产生最多框数
    infer_post_nms_top_n: 1000 #推理时经后处理后产生最多框数
  proposal_target_generator: #目标框后处理
    name: ProposalTargetGenerator #目标框后处理类名
    batch_size_per_im: 512 #图片尺寸
    bbox_reg_weights: [0.1, 0.1, 0.2, 0.2] #bbox权重
    bg_thresh_hi: [0.5,] #最大阈值 在bg_thresh_hi、bg_thresh_lo区间为背景
    bg_thresh_lo: [0.0,] #最小阈值 在bg_thresh_hi、bg_thresh_lo区间为背景
    fg_thresh: [0.5,] #被选正例iou阈值     
    fg_fraction: 0.25 #类别阈值   
'''

相关引用库：

import numpy as np
import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from ppdet.core.workspace import register
from . import ops

Anchor类

@register
class Anchor(object):
    __inject__ = ['anchor_generator', 'anchor_target_generator']

    def __init__(self, anchor_generator, anchor_target_generator):
        super(Anchor, self).__init__()
        self.anchor_generator = anchor_generator
        self.anchor_target_generator = anchor_target_generator

    def __call__(self, rpn_feats):
        anchors = []
        num_level = len(rpn_feats)
        for i, rpn_feat in enumerate(rpn_feats):
            anchor, var = self.anchor_generator(rpn_feat, i)
            anchors.append((anchor, var))
        return anchors
    # 具体计算实现
    def _get_target_input(self, rpn_feats, anchors):
        rpn_score_list = []
        rpn_delta_list = []
        anchor_list = []
        for (rpn_score, rpn_delta), (anchor, var) in zip(rpn_feats, anchors):
            rpn_score = paddle.transpose(rpn_score, perm=[0, 2, 3, 1])
            rpn_delta = paddle.transpose(rpn_delta, perm=[0, 2, 3, 1])
            rpn_score = paddle.reshape(x=rpn_score, shape=(0, -1, 1))
            rpn_delta = paddle.reshape(x=rpn_delta, shape=(0, -1, 4))

            anchor = paddle.reshape(anchor, shape=(-1, 4))
            var = paddle.reshape(var, shape=(-1, 4))
            rpn_score_list.append(rpn_score)
            rpn_delta_list.append(rpn_delta)
            anchor_list.append(anchor)

        rpn_scores = paddle.concat(rpn_score_list, axis=1)
        rpn_deltas = paddle.concat(rpn_delta_list, axis=1)
        anchors = paddle.concat(anchor_list)
        return rpn_scores, rpn_deltas, anchors
    # 得出候选框的信息，用于计算RPN loss
    def generate_loss_inputs(self, inputs, rpn_head_out, anchors):
        if len(rpn_head_out) != len(anchors):
            raise ValueError(
                "rpn_head_out and anchors should have same length, "
                " but received rpn_head_out' length is {} and anchors' "
                " length is {}".format(len(rpn_head_out), len(anchors)))
        rpn_score, rpn_delta, anchors = self._get_target_input(rpn_head_out,
                                                               anchors)

        score_pred, roi_pred, score_tgt, roi_tgt, roi_weight = self.anchor_target_generator(
            bbox_pred=rpn_delta,
            cls_logits=rpn_score,
            anchor_box=anchors,
            gt_boxes=inputs['gt_bbox'],
            is_crowd=inputs['is_crowd'],
            im_info=inputs['im_info'])
        outs = {
            'rpn_score_pred': score_pred,
            'rpn_score_target': score_tgt,
            'rpn_rois_pred': roi_pred,
            'rpn_rois_target': roi_tgt,
            'rpn_rois_weight': roi_weight
        }
        return outs

Proposal 类，产生ROI

@register
class Proposal(object):
    __inject__ = ['proposal_generator', 'proposal_target_generator']

    def __init__(self, proposal_generator, proposal_target_generator):
        super(Proposal, self).__init__()
        self.proposal_generator = proposal_generator
        self.proposal_target_generator = proposal_target_generator

    def generate_proposal(self, inputs, rpn_head_out, anchor_out):
        # TODO: delete im_info 
        try:
            im_shape = inputs['im_info']
        except:
            im_shape = inputs['im_shape']
        rpn_rois_list = []
        rpn_prob_list = []
        rpn_rois_num_list = []
        for (rpn_score, rpn_delta), (anchor, var) in zip(rpn_head_out,
                                                         anchor_out):
            rpn_prob = F.sigmoid(rpn_score)
            rpn_rois, rpn_rois_prob, rpn_rois_num, post_nms_top_n = self.proposal_generator(
                scores=rpn_prob,
                bbox_deltas=rpn_delta,
                anchors=anchor,
                variances=var,
                im_shape=im_shape,
                mode=inputs['mode'])
            if len(rpn_head_out) == 1:
                return rpn_rois, rpn_rois_num
            rpn_rois_list.append(rpn_rois)
            rpn_prob_list.append(rpn_rois_prob)
            rpn_rois_num_list.append(rpn_rois_num)

        start_level = 2
        end_level = start_level + len(rpn_head_out)
        rois_collect, rois_num_collect = ops.collect_fpn_proposals(
            rpn_rois_list,
            rpn_prob_list,
            start_level,
            end_level,
            post_nms_top_n,
            rois_num_per_level=rpn_rois_num_list)
        return rois_collect, rois_num_collect
    #生成目标ROI
    def generate_proposal_target(self,
                                 inputs,
                                 rois,
                                 rois_num,
                                 stage=0,
                                 max_overlap=None):
        outs = self.proposal_target_generator(
            rpn_rois=rois,
            rpn_rois_num=rois_num,
            gt_classes=inputs['gt_class'],
            is_crowd=inputs['is_crowd'],
            gt_boxes=inputs['gt_bbox'],
            im_info=inputs['im_info'],
            stage=stage,
            max_overlap=max_overlap)
        rois = outs[0]
        max_overlap = outs[-1]
        rois_num = outs[-2]
        targets = {
            'labels_int32': outs[1],
            'bbox_targets': outs[2],
            'bbox_inside_weights': outs[3],
            'bbox_outside_weights': outs[4]
        }
        return rois, rois_num, targets, max_overlap
    #bbox 精炼
    def refine_bbox(self, roi, bbox_delta, stage=1):
        out_dim = bbox_delta.shape[1] // 4
        bbox_delta_r = paddle.reshape(bbox_delta, (-1, out_dim, 4))
        bbox_delta_s = paddle.slice(
            bbox_delta_r, axes=[1], starts=[1], ends=[2])

        reg_weights = [
            i / stage for i in self.proposal_target_generator.bbox_reg_weights
        ]
        refined_bbox = ops.box_coder(
            prior_box=roi,
            prior_box_var=reg_weights,
            target_box=bbox_delta_s,
            code_type='decode_center_size',
            box_normalized=False,
            axis=1)
        refined_bbox = paddle.reshape(refined_bbox, shape=[-1, 4])
        return refined_bbox

    def __call__(self,
                 inputs,
                 rpn_head_out,
                 anchor_out,
                 stage=0,
                 proposal_out=None,
                 bbox_head_out=None,
                 max_overlap=None):
        if stage == 0:
            roi, rois_num = self.generate_proposal(inputs, rpn_head_out,
                                                   anchor_out)
            self.targets_list = []
            self.max_overlap = None

        else:
            bbox_delta = bbox_head_out[1]
            roi = self.refine_bbox(proposal_out[0], bbox_delta, stage)
            rois_num = proposal_out[1]
        if inputs['mode'] == 'train':
            roi, rois_num, targets, self.max_overlap = self.generate_proposal_target(
                inputs, roi, rois_num, stage, self.max_overlap)
            self.targets_list.append(targets)
        return roi, rois_num

    def get_targets(self):
        return self.targets_list

    def get_max_overlap(self):
        return self.max_overlap

 modeling/head/bbox_head.py解读：

配置文件解析：

BBoxHead: #BBox头
  bbox_feat: #bbox特征
    name: BBoxFeat  #bbox特征类名
    roi_extractor:  #roi提取
      name: RoIAlign #RoIAlign
      resolution: 7 #像素值
      sampling_ratio: 2 #采样率
    head_feat:  #head特征
      name: TwoFCHead  #就是两个全连接类名
      in_dim: 256  #输入维度
      mlp_dim: 1024  #中间维度
  in_feat: 1024  #特征维度

 相关引用库：

import paddle
from paddle import ParamAttr
import paddle.nn as nn
import paddle.nn.functional as F
from paddle.nn import ReLU
from paddle.nn.initializer import Normal, XavierUniform
from paddle.regularizer import L2Decay
from ppdet.core.workspace import register
from ppdet.modeling import ops

from ..backbone.name_adapter import NameAdapter
from ..backbone.resnet import Blocks

两个全连接层用来给BBox做回归、分类用

@register
class TwoFCHead(nn.Layer):

    __shared__ = ['roi_stages']

    def __init__(self, in_dim=256, mlp_dim=1024, resolution=7, roi_stages=1):
        super(TwoFCHead, self).__init__()
        self.in_dim = in_dim
        self.mlp_dim = mlp_dim
        self.roi_stages = roi_stages
        fan = in_dim * resolution * resolution
        self.fc6_list = []
        self.fc6_relu_list = []
        self.fc7_list = []
        self.fc7_relu_list = []
        for stage in range(roi_stages):
            fc6_name = 'fc6_{}'.format(stage)
            fc7_name = 'fc7_{}'.format(stage)
            lr_factor = 2**stage
            fc6 = self.add_sublayer(
                fc6_name,
                nn.Linear(
                    in_dim * resolution * resolution,
                    mlp_dim,
                    weight_attr=ParamAttr(
                        learning_rate=lr_factor,
                        initializer=XavierUniform(fan_out=fan)),
                    bias_attr=ParamAttr(
                        learning_rate=2. * lr_factor, regularizer=L2Decay(0.))))
            fc6_relu = self.add_sublayer(fc6_name + 'act', ReLU())
            fc7 = self.add_sublayer(
                fc7_name,
                nn.Linear(
                    mlp_dim,
                    mlp_dim,
                    weight_attr=ParamAttr(
                        learning_rate=lr_factor, initializer=XavierUniform()),
                    bias_attr=ParamAttr(
                        learning_rate=2. * lr_factor, regularizer=L2Decay(0.))))
            fc7_relu = self.add_sublayer(fc7_name + 'act', ReLU())
            self.fc6_list.append(fc6)
            self.fc6_relu_list.append(fc6_relu)
            self.fc7_list.append(fc7)
            self.fc7_relu_list.append(fc7_relu)

    def forward(self, rois_feat, stage=0):
        rois_feat = paddle.flatten(rois_feat, start_axis=1, stop_axis=-1)
        fc6 = self.fc6_list[stage](rois_feat)
        fc6_relu = self.fc6_relu_list[stage](fc6)
        fc7 = self.fc7_list[stage](fc6_relu)
        fc7_relu = self.fc7_relu_list[stage](fc7)
        return fc7_relu

BBbox特征类

@register
class BBoxFeat(nn.Layer):
    __inject__ = ['roi_extractor', 'head_feat']

    def __init__(self, roi_extractor, head_feat):
        super(BBoxFeat, self).__init__()
        self.roi_extractor = roi_extractor
        self.head_feat = head_feat
        self.rois_feat_list = []

    def forward(self, body_feats, rois, spatial_scale, stage=0):
        rois_feat = self.roi_extractor(body_feats, rois, spatial_scale)
        bbox_feat = self.head_feat(rois_feat, stage)
        return rois_feat, bbox_feat

 BBox头类

@register
class BBoxHead(nn.Layer):
    __shared__ = ['num_classes', 'roi_stages']
    __inject__ = ['bbox_feat']

    def __init__(self,
                 bbox_feat,
                 in_feat=1024,
                 num_classes=81,
                 cls_agnostic=False,
                 roi_stages=1,
                 with_pool=False,
                 score_stage=[0, 1, 2],
                 delta_stage=[2]):
        super(BBoxHead, self).__init__()
        self.num_classes = num_classes
        self.cls_agnostic = cls_agnostic
        self.delta_dim = 2 if cls_agnostic else num_classes
        self.bbox_feat = bbox_feat
        self.roi_stages = roi_stages
        self.bbox_score_list = []
        self.bbox_delta_list = []
        self.roi_feat_list = [[] for i in range(roi_stages)]
        self.with_pool = with_pool
        self.score_stage = score_stage
        self.delta_stage = delta_stage
        for stage in range(roi_stages):
            score_name = 'bbox_score_{}'.format(stage)
            delta_name = 'bbox_delta_{}'.format(stage)
            lr_factor = 2**stage
            bbox_score = self.add_sublayer(
                score_name,
                nn.Linear(
                    in_feat,
                    1 * self.num_classes,
                    weight_attr=ParamAttr(
                        learning_rate=lr_factor,
                        initializer=Normal(
                            mean=0.0, std=0.01)),
                    bias_attr=ParamAttr(
                        learning_rate=2. * lr_factor, regularizer=L2Decay(0.))))

            bbox_delta = self.add_sublayer(
                delta_name,
                nn.Linear(
                    in_feat,
                    4 * self.delta_dim,
                    weight_attr=ParamAttr(
                        learning_rate=lr_factor,
                        initializer=Normal(
                            mean=0.0, std=0.001)),
                    bias_attr=ParamAttr(
                        learning_rate=2. * lr_factor, regularizer=L2Decay(0.))))
            self.bbox_score_list.append(bbox_score)
            self.bbox_delta_list.append(bbox_delta)

    def forward(self,
                body_feats=None,
                rois=None,
                spatial_scale=None,
                stage=0,
                roi_stage=-1):
        if rois is not None:
            rois_feat, bbox_feat = self.bbox_feat(body_feats, rois,
                                                  spatial_scale, stage)
            self.roi_feat_list[stage] = rois_feat
        else:
            rois_feat = self.roi_feat_list[roi_stage]
            bbox_feat = self.bbox_feat.head_feat(rois_feat, stage)
        if self.with_pool:
            bbox_feat_ = F.adaptive_avg_pool2d(bbox_feat, output_size=1)
            bbox_feat_ = paddle.squeeze(bbox_feat_, axis=[2, 3])
            scores = self.bbox_score_list[stage](bbox_feat_)
            deltas = self.bbox_delta_list[stage](bbox_feat_)
        else:
            scores = self.bbox_score_list[stage](bbox_feat)
            deltas = self.bbox_delta_list[stage](bbox_feat)
        bbox_head_out = (scores, deltas)
        return bbox_feat, bbox_head_out, self.bbox_feat.head_feat
    # loss具体计算
    def _get_head_loss(self, score, delta, target):
        # bbox cls  
        labels_int64 = paddle.cast(x=target['labels_int32'], dtype='int64')
        labels_int64.stop_gradient = True
        loss_bbox_cls = F.softmax_with_cross_entropy(
            logits=score, label=labels_int64)
        loss_bbox_cls = paddle.mean(loss_bbox_cls)
        # bbox reg
        loss_bbox_reg = ops.smooth_l1(
            input=delta,
            label=target['bbox_targets'],
            inside_weight=target['bbox_inside_weights'],
            outside_weight=target['bbox_outside_weights'],
            sigma=1.0)
        loss_bbox_reg = paddle.mean(loss_bbox_reg)
        return loss_bbox_cls, loss_bbox_reg
    #计算 分类loss  位置loss
    def get_loss(self, bbox_head_out, targets):
        loss_bbox = {}
        cls_name = 'loss_bbox_cls'
        reg_name = 'loss_bbox_reg'
        for lvl, (bboxhead, target) in enumerate(zip(bbox_head_out, targets)):
            score, delta = bboxhead
            if len(targets) > 1:
                cls_name = 'loss_bbox_cls_{}'.format(lvl)
                reg_name = 'loss_bbox_reg_{}'.format(lvl)
            loss_bbox_cls, loss_bbox_reg = self._get_head_loss(score, delta,
                                                               target)
            loss_weight = 1. / 2**lvl
            loss_bbox[cls_name] = loss_bbox_cls * loss_weight
            loss_bbox[reg_name] = loss_bbox_reg * loss_weight
        return loss_bbox
    #用于bbox预测
    def get_prediction(self, bbox_head_out, rois):
        proposal, proposal_num = rois
        score, delta = bbox_head_out
        bbox_prob = F.softmax(score)
        delta = paddle.reshape(delta, (-1, self.delta_dim, 4))
        bbox_pred = (delta, bbox_prob)
        return bbox_pred, rois
    #级联预测
    def get_cascade_prediction(self, bbox_head_out, rois):
        proposal_list = []
        prob_list = []
        delta_list = []
        for stage in range(len(rois)):
            proposals = rois[stage]
            bboxhead = bbox_head_out[stage]
            score, delta = bboxhead
            proposal, proposal_num = proposals
            if stage in self.score_stage:
                if stage < 2:
                    _, head_out, _ = self(stage=stage, roi_stage=-1)
                    score = head_out[0]

                bbox_prob = F.softmax(score)
                prob_list.append(bbox_prob)
            if stage in self.delta_stage:
                proposal_list.append(proposal)
                delta_list.append(delta)
        bbox_prob = paddle.mean(paddle.stack(prob_list), axis=0)
        delta = paddle.mean(paddle.stack(delta_list), axis=0)
        proposal = paddle.mean(paddle.stack(proposal_list), axis=0)
        delta = paddle.reshape(delta, (-1, self.delta_dim, 4))
        if self.cls_agnostic:
            N, C, M = delta.shape
            delta = delta[:, 1:2, :]
            delta = paddle.expand(delta, [N, self.num_classes, M])
        bboxes = (proposal, proposal_num)
        bbox_pred = (delta, bbox_prob)
        return bbox_pred, bboxes