2021SC@SDUSC
本文章为PaddleDetection-MaskRcnn的结构拆分
首先是Head部分,算法结构图:
首先是在yaml的配置文件:/configs/_base_/models/mask_rcnn_r50_fpn.yml
'''
RPNHead: #RPNHead类名
rpn_feat: #rpn特征
name: RPNFeat #类名
feat_in: 256 #输入通道数
feat_out: 256 #输出通道数
anchor_per_position: 3 #anchor个数
rpn_channel: 256 #rpn通道数
'''
相关引用库如下:
import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from paddle import ParamAttr
from paddle.nn.initializer import Normal
from paddle.regularizer import L2Decay
from paddle.nn import Conv2D
from ppdet.core.workspace import register
from ppdet.modeling import ops
候选框网络特征类:
@register
class RPNFeat(nn.Layer):
def __init__(self, feat_in=1024, feat_out=1024):
super(RPNFeat, self).__init__()
# rpn feat is shared with each level
self.rpn_conv = Conv2D(
in_channels=feat_in,
out_channels=feat_out,
kernel_size=3,
padding=1,
weight_attr=ParamAttr(initializer=Normal(
mean=0., std=0.01)),
bias_attr=ParamAttr(
learning_rate=2., regularizer=L2Decay(0.)))
def forward(self, inputs, feats):
rpn_feats = []
for feat in feats:
rpn_feats.append(F.relu(self.rpn_conv(feat)))
return rpn_feats
候选框头
@register
class RPNHead(nn.Layer):
__inject__ = ['rpn_feat']
def __init__(self, rpn_feat, anchor_per_position=15, rpn_channel=1024):
super(RPNHead, self).__init__()
self.rpn_feat = rpn_feat
if isinstance(rpn_feat, dict):
self.rpn_feat = RPNFeat(**rpn_feat)
# rpn head is shared with each level
# rpn roi classification scores
self.rpn_rois_score = Conv2D(
in_channels=rpn_channel,
out_channels=anchor_per_position,
kernel_size=1,
padding=0,
weight_attr=ParamAttr(initializer=Normal(
mean=0., std=0.01)),
bias_attr=ParamAttr(
learning_rate=2., regularizer=L2Decay(0.)))
# rpn roi bbox regression deltas
self.rpn_rois_delta = Conv2D(
in_channels=rpn_channel,
out_channels=4 * anchor_per_position,
kernel_size=1,
padding=0,
weight_attr=ParamAttr(initializer=Normal(
mean=0., std=0.01)),
bias_attr=ParamAttr(
learning_rate=2., regularizer=L2Decay(0.)))
def forward(self, inputs, feats):
rpn_feats = self.rpn_feat(inputs, feats)
rpn_head_out = []
for rpn_feat in rpn_feats:
rrs = self.rpn_rois_score(rpn_feat)
rrd = self.rpn_rois_delta(rpn_feat)
rpn_head_out.append((rrs, rrd))
return rpn_feats, rpn_head_out
#计算RPN loss
def get_loss(self, loss_inputs):
# cls loss
score_tgt = paddle.cast(
x=loss_inputs['rpn_score_target'], dtype='float32')
score_tgt.stop_gradient = True
loss_rpn_cls = ops.sigmoid_cross_entropy_with_logits(
input=loss_inputs['rpn_score_pred'], label=score_tgt)
loss_rpn_cls = paddle.mean(loss_rpn_cls, name='loss_rpn_cls')
# reg loss
loc_tgt = paddle.cast(x=loss_inputs['rpn_rois_target'], dtype='float32')
loc_tgt.stop_gradient = True
loss_rpn_reg = ops.smooth_l1(
input=loss_inputs['rpn_rois_pred'],
label=loc_tgt,
inside_weight=loss_inputs['rpn_rois_weight'],
outside_weight=loss_inputs['rpn_rois_weight'],
sigma=3.0, )
loss_rpn_reg = paddle.sum(loss_rpn_reg)
score_shape = paddle.shape(score_tgt)
score_shape = paddle.cast(score_shape, dtype='float32')
norm = paddle.prod(score_shape)
norm.stop_gradient = True
loss_rpn_reg = loss_rpn_reg / norm
return {'loss_rpn_cls': loss_rpn_cls, 'loss_rpn_reg': loss_rpn_reg}
定义了初始化方法,还有前向传播函数的实现,以及获得损失的方法。
modeling/bbox.py源码解析:
配置文件解析:
'''
Anchor:#锚框
anchor_generator: #产生锚框
name: AnchorGeneratorRPN #类名
aspect_ratios: [0.5, 1.0, 2.0] #锚框比例
anchor_start_size: 32 #产生锚框尺寸大小
stride: [4., 4.] #步长
anchor_target_generator: #目标框
name: AnchorTargetGeneratorRPN #产生目标框类名
batch_size_per_im: 256 #输入图片尺寸
fg_fraction: 0.5 #RPN阈值
negative_overlap: 0.3 #负例iou置信度
positive_overlap: 0.7 #正例iou置信度
straddle_thresh: 0.0 #目标框阈值
Proposal: #目标框
proposal_generator: #生成目标框
name: ProposalGenerator #生成目标框类名
min_size: 0.0 #最小尺寸
nms_thresh: 0.7 #nms阈值
train_pre_nms_top_n: 2000 #训练时产生最多框数
train_post_nms_top_n: 2000 #训练时经后处理后产生最多框数
infer_pre_nms_top_n: 1000 #推理时产生最多框数
infer_post_nms_top_n: 1000 #推理时经后处理后产生最多框数
proposal_target_generator: #目标框后处理
name: ProposalTargetGenerator #目标框后处理类名
batch_size_per_im: 512 #图片尺寸
bbox_reg_weights: [0.1, 0.1, 0.2, 0.2] #bbox权重
bg_thresh_hi: [0.5,] #最大阈值 在bg_thresh_hi、bg_thresh_lo区间为背景
bg_thresh_lo: [0.0,] #最小阈值 在bg_thresh_hi、bg_thresh_lo区间为背景
fg_thresh: [0.5,] #被选正例iou阈值
fg_fraction: 0.25 #类别阈值
'''
相关引用库:
import numpy as np
import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from ppdet.core.workspace import register
from . import ops
Anchor类
@register
class Anchor(object):
__inject__ = ['anchor_generator', 'anchor_target_generator']
def __init__(self, anchor_generator, anchor_target_generator):
super(Anchor, self).__init__()
self.anchor_generator = anchor_generator
self.anchor_target_generator = anchor_target_generator
def __call__(self, rpn_feats):
anchors = []
num_level = len(rpn_feats)
for i, rpn_feat in enumerate(rpn_feats):
anchor, var = self.anchor_generator(rpn_feat, i)
anchors.append((anchor, var))
return anchors
# 具体计算实现
def _get_target_input(self, rpn_feats, anchors):
rpn_score_list = []
rpn_delta_list = []
anchor_list = []
for (rpn_score, rpn_delta), (anchor, var) in zip(rpn_feats, anchors):
rpn_score = paddle.transpose(rpn_score, perm=[0, 2, 3, 1])
rpn_delta = paddle.transpose(rpn_delta, perm=[0, 2, 3, 1])
rpn_score = paddle.reshape(x=rpn_score, shape=(0, -1, 1))
rpn_delta = paddle.reshape(x=rpn_delta, shape=(0, -1, 4))
anchor = paddle.reshape(anchor, shape=(-1, 4))
var = paddle.reshape(var, shape=(-1, 4))
rpn_score_list.append(rpn_score)
rpn_delta_list.append(rpn_delta)
anchor_list.append(anchor)
rpn_scores = paddle.concat(rpn_score_list, axis=1)
rpn_deltas = paddle.concat(rpn_delta_list, axis=1)
anchors = paddle.concat(anchor_list)
return rpn_scores, rpn_deltas, anchors
# 得出候选框的信息,用于计算RPN loss
def generate_loss_inputs(self, inputs, rpn_head_out, anchors):
if len(rpn_head_out) != len(anchors):
raise ValueError(
"rpn_head_out and anchors should have same length, "
" but received rpn_head_out' length is {} and anchors' "
" length is {}".format(len(rpn_head_out), len(anchors)))
rpn_score, rpn_delta, anchors = self._get_target_input(rpn_head_out,
anchors)
score_pred, roi_pred, score_tgt, roi_tgt, roi_weight = self.anchor_target_generator(
bbox_pred=rpn_delta,
cls_logits=rpn_score,
anchor_box=anchors,
gt_boxes=inputs['gt_bbox'],
is_crowd=inputs['is_crowd'],
im_info=inputs['im_info'])
outs = {
'rpn_score_pred': score_pred,
'rpn_score_target': score_tgt,
'rpn_rois_pred': roi_pred,
'rpn_rois_target': roi_tgt,
'rpn_rois_weight': roi_weight
}
return outs
Proposal 类,产生ROI
@register
class Proposal(object):
__inject__ = ['proposal_generator', 'proposal_target_generator']
def __init__(self, proposal_generator, proposal_target_generator):
super(Proposal, self).__init__()
self.proposal_generator = proposal_generator
self.proposal_target_generator = proposal_target_generator
def generate_proposal(self, inputs, rpn_head_out, anchor_out):
# TODO: delete im_info
try:
im_shape = inputs['im_info']
except:
im_shape = inputs['im_shape']
rpn_rois_list = []
rpn_prob_list = []
rpn_rois_num_list = []
for (rpn_score, rpn_delta), (anchor, var) in zip(rpn_head_out,
anchor_out):
rpn_prob = F.sigmoid(rpn_score)
rpn_rois, rpn_rois_prob, rpn_rois_num, post_nms_top_n = self.proposal_generator(
scores=rpn_prob,
bbox_deltas=rpn_delta,
anchors=anchor,
variances=var,
im_shape=im_shape,
mode=inputs['mode'])
if len(rpn_head_out) == 1:
return rpn_rois, rpn_rois_num
rpn_rois_list.append(rpn_rois)
rpn_prob_list.append(rpn_rois_prob)
rpn_rois_num_list.append(rpn_rois_num)
start_level = 2
end_level = start_level + len(rpn_head_out)
rois_collect, rois_num_collect = ops.collect_fpn_proposals(
rpn_rois_list,
rpn_prob_list,
start_level,
end_level,
post_nms_top_n,
rois_num_per_level=rpn_rois_num_list)
return rois_collect, rois_num_collect
#生成目标ROI
def generate_proposal_target(self,
inputs,
rois,
rois_num,
stage=0,
max_overlap=None):
outs = self.proposal_target_generator(
rpn_rois=rois,
rpn_rois_num=rois_num,
gt_classes=inputs['gt_class'],
is_crowd=inputs['is_crowd'],
gt_boxes=inputs['gt_bbox'],
im_info=inputs['im_info'],
stage=stage,
max_overlap=max_overlap)
rois = outs[0]
max_overlap = outs[-1]
rois_num = outs[-2]
targets = {
'labels_int32': outs[1],
'bbox_targets': outs[2],
'bbox_inside_weights': outs[3],
'bbox_outside_weights': outs[4]
}
return rois, rois_num, targets, max_overlap
#bbox 精炼
def refine_bbox(self, roi, bbox_delta, stage=1):
out_dim = bbox_delta.shape[1] // 4
bbox_delta_r = paddle.reshape(bbox_delta, (-1, out_dim, 4))
bbox_delta_s = paddle.slice(
bbox_delta_r, axes=[1], starts=[1], ends=[2])
reg_weights = [
i / stage for i in self.proposal_target_generator.bbox_reg_weights
]
refined_bbox = ops.box_coder(
prior_box=roi,
prior_box_var=reg_weights,
target_box=bbox_delta_s,
code_type='decode_center_size',
box_normalized=False,
axis=1)
refined_bbox = paddle.reshape(refined_bbox, shape=[-1, 4])
return refined_bbox
def __call__(self,
inputs,
rpn_head_out,
anchor_out,
stage=0,
proposal_out=None,
bbox_head_out=None,
max_overlap=None):
if stage == 0:
roi, rois_num = self.generate_proposal(inputs, rpn_head_out,
anchor_out)
self.targets_list = []
self.max_overlap = None
else:
bbox_delta = bbox_head_out[1]
roi = self.refine_bbox(proposal_out[0], bbox_delta, stage)
rois_num = proposal_out[1]
if inputs['mode'] == 'train':
roi, rois_num, targets, self.max_overlap = self.generate_proposal_target(
inputs, roi, rois_num, stage, self.max_overlap)
self.targets_list.append(targets)
return roi, rois_num
def get_targets(self):
return self.targets_list
def get_max_overlap(self):
return self.max_overlap
modeling/head/bbox_head.py解读:
配置文件解析:
BBoxHead: #BBox头
bbox_feat: #bbox特征
name: BBoxFeat #bbox特征类名
roi_extractor: #roi提取
name: RoIAlign #RoIAlign
resolution: 7 #像素值
sampling_ratio: 2 #采样率
head_feat: #head特征
name: TwoFCHead #就是两个全连接类名
in_dim: 256 #输入维度
mlp_dim: 1024 #中间维度
in_feat: 1024 #特征维度
相关引用库:
import paddle
from paddle import ParamAttr
import paddle.nn as nn
import paddle.nn.functional as F
from paddle.nn import ReLU
from paddle.nn.initializer import Normal, XavierUniform
from paddle.regularizer import L2Decay
from ppdet.core.workspace import register
from ppdet.modeling import ops
from ..backbone.name_adapter import NameAdapter
from ..backbone.resnet import Blocks
两个全连接层用来给BBox做回归、分类用
@register
class TwoFCHead(nn.Layer):
__shared__ = ['roi_stages']
def __init__(self, in_dim=256, mlp_dim=1024, resolution=7, roi_stages=1):
super(TwoFCHead, self).__init__()
self.in_dim = in_dim
self.mlp_dim = mlp_dim
self.roi_stages = roi_stages
fan = in_dim * resolution * resolution
self.fc6_list = []
self.fc6_relu_list = []
self.fc7_list = []
self.fc7_relu_list = []
for stage in range(roi_stages):
fc6_name = 'fc6_{}'.format(stage)
fc7_name = 'fc7_{}'.format(stage)
lr_factor = 2**stage
fc6 = self.add_sublayer(
fc6_name,
nn.Linear(
in_dim * resolution * resolution,
mlp_dim,
weight_attr=ParamAttr(
learning_rate=lr_factor,
initializer=XavierUniform(fan_out=fan)),
bias_attr=ParamAttr(
learning_rate=2. * lr_factor, regularizer=L2Decay(0.))))
fc6_relu = self.add_sublayer(fc6_name + 'act', ReLU())
fc7 = self.add_sublayer(
fc7_name,
nn.Linear(
mlp_dim,
mlp_dim,
weight_attr=ParamAttr(
learning_rate=lr_factor, initializer=XavierUniform()),
bias_attr=ParamAttr(
learning_rate=2. * lr_factor, regularizer=L2Decay(0.))))
fc7_relu = self.add_sublayer(fc7_name + 'act', ReLU())
self.fc6_list.append(fc6)
self.fc6_relu_list.append(fc6_relu)
self.fc7_list.append(fc7)
self.fc7_relu_list.append(fc7_relu)
def forward(self, rois_feat, stage=0):
rois_feat = paddle.flatten(rois_feat, start_axis=1, stop_axis=-1)
fc6 = self.fc6_list[stage](rois_feat)
fc6_relu = self.fc6_relu_list[stage](fc6)
fc7 = self.fc7_list[stage](fc6_relu)
fc7_relu = self.fc7_relu_list[stage](fc7)
return fc7_relu
BBbox特征类
@register
class BBoxFeat(nn.Layer):
__inject__ = ['roi_extractor', 'head_feat']
def __init__(self, roi_extractor, head_feat):
super(BBoxFeat, self).__init__()
self.roi_extractor = roi_extractor
self.head_feat = head_feat
self.rois_feat_list = []
def forward(self, body_feats, rois, spatial_scale, stage=0):
rois_feat = self.roi_extractor(body_feats, rois, spatial_scale)
bbox_feat = self.head_feat(rois_feat, stage)
return rois_feat, bbox_feat
BBox头类
@register
class BBoxHead(nn.Layer):
__shared__ = ['num_classes', 'roi_stages']
__inject__ = ['bbox_feat']
def __init__(self,
bbox_feat,
in_feat=1024,
num_classes=81,
cls_agnostic=False,
roi_stages=1,
with_pool=False,
score_stage=[0, 1, 2],
delta_stage=[2]):
super(BBoxHead, self).__init__()
self.num_classes = num_classes
self.cls_agnostic = cls_agnostic
self.delta_dim = 2 if cls_agnostic else num_classes
self.bbox_feat = bbox_feat
self.roi_stages = roi_stages
self.bbox_score_list = []
self.bbox_delta_list = []
self.roi_feat_list = [[] for i in range(roi_stages)]
self.with_pool = with_pool
self.score_stage = score_stage
self.delta_stage = delta_stage
for stage in range(roi_stages):
score_name = 'bbox_score_{}'.format(stage)
delta_name = 'bbox_delta_{}'.format(stage)
lr_factor = 2**stage
bbox_score = self.add_sublayer(
score_name,
nn.Linear(
in_feat,
1 * self.num_classes,
weight_attr=ParamAttr(
learning_rate=lr_factor,
initializer=Normal(
mean=0.0, std=0.01)),
bias_attr=ParamAttr(
learning_rate=2. * lr_factor, regularizer=L2Decay(0.))))
bbox_delta = self.add_sublayer(
delta_name,
nn.Linear(
in_feat,
4 * self.delta_dim,
weight_attr=ParamAttr(
learning_rate=lr_factor,
initializer=Normal(
mean=0.0, std=0.001)),
bias_attr=ParamAttr(
learning_rate=2. * lr_factor, regularizer=L2Decay(0.))))
self.bbox_score_list.append(bbox_score)
self.bbox_delta_list.append(bbox_delta)
def forward(self,
body_feats=None,
rois=None,
spatial_scale=None,
stage=0,
roi_stage=-1):
if rois is not None:
rois_feat, bbox_feat = self.bbox_feat(body_feats, rois,
spatial_scale, stage)
self.roi_feat_list[stage] = rois_feat
else:
rois_feat = self.roi_feat_list[roi_stage]
bbox_feat = self.bbox_feat.head_feat(rois_feat, stage)
if self.with_pool:
bbox_feat_ = F.adaptive_avg_pool2d(bbox_feat, output_size=1)
bbox_feat_ = paddle.squeeze(bbox_feat_, axis=[2, 3])
scores = self.bbox_score_list[stage](bbox_feat_)
deltas = self.bbox_delta_list[stage](bbox_feat_)
else:
scores = self.bbox_score_list[stage](bbox_feat)
deltas = self.bbox_delta_list[stage](bbox_feat)
bbox_head_out = (scores, deltas)
return bbox_feat, bbox_head_out, self.bbox_feat.head_feat
# loss具体计算
def _get_head_loss(self, score, delta, target):
# bbox cls
labels_int64 = paddle.cast(x=target['labels_int32'], dtype='int64')
labels_int64.stop_gradient = True
loss_bbox_cls = F.softmax_with_cross_entropy(
logits=score, label=labels_int64)
loss_bbox_cls = paddle.mean(loss_bbox_cls)
# bbox reg
loss_bbox_reg = ops.smooth_l1(
input=delta,
label=target['bbox_targets'],
inside_weight=target['bbox_inside_weights'],
outside_weight=target['bbox_outside_weights'],
sigma=1.0)
loss_bbox_reg = paddle.mean(loss_bbox_reg)
return loss_bbox_cls, loss_bbox_reg
#计算 分类loss 位置loss
def get_loss(self, bbox_head_out, targets):
loss_bbox = {}
cls_name = 'loss_bbox_cls'
reg_name = 'loss_bbox_reg'
for lvl, (bboxhead, target) in enumerate(zip(bbox_head_out, targets)):
score, delta = bboxhead
if len(targets) > 1:
cls_name = 'loss_bbox_cls_{}'.format(lvl)
reg_name = 'loss_bbox_reg_{}'.format(lvl)
loss_bbox_cls, loss_bbox_reg = self._get_head_loss(score, delta,
target)
loss_weight = 1. / 2**lvl
loss_bbox[cls_name] = loss_bbox_cls * loss_weight
loss_bbox[reg_name] = loss_bbox_reg * loss_weight
return loss_bbox
#用于bbox预测
def get_prediction(self, bbox_head_out, rois):
proposal, proposal_num = rois
score, delta = bbox_head_out
bbox_prob = F.softmax(score)
delta = paddle.reshape(delta, (-1, self.delta_dim, 4))
bbox_pred = (delta, bbox_prob)
return bbox_pred, rois
#级联预测
def get_cascade_prediction(self, bbox_head_out, rois):
proposal_list = []
prob_list = []
delta_list = []
for stage in range(len(rois)):
proposals = rois[stage]
bboxhead = bbox_head_out[stage]
score, delta = bboxhead
proposal, proposal_num = proposals
if stage in self.score_stage:
if stage < 2:
_, head_out, _ = self(stage=stage, roi_stage=-1)
score = head_out[0]
bbox_prob = F.softmax(score)
prob_list.append(bbox_prob)
if stage in self.delta_stage:
proposal_list.append(proposal)
delta_list.append(delta)
bbox_prob = paddle.mean(paddle.stack(prob_list), axis=0)
delta = paddle.mean(paddle.stack(delta_list), axis=0)
proposal = paddle.mean(paddle.stack(proposal_list), axis=0)
delta = paddle.reshape(delta, (-1, self.delta_dim, 4))
if self.cls_agnostic:
N, C, M = delta.shape
delta = delta[:, 1:2, :]
delta = paddle.expand(delta, [N, self.num_classes, M])
bboxes = (proposal, proposal_num)
bbox_pred = (delta, bbox_prob)
return bbox_pred, bboxes