pytorch faster rcnn softnms
frcnn使用softnms方法一:pytorch复现版本的cpu版softnms(本方法可以跑通)
划重点:模型无需重新训练,仅在inference用即可提点
提点结果:
(baseline:pytorch版的frcnn,github地址:https://github.com/jwyang/faster-rcnn.pytorch)
- kitti,vgg16
- pedestrian提高4.5个点
- car提高4.5个点
- voc07,vgg16
- person提高1个点
- car提高4个点
- voc0712,res101
- 几乎不变,有0.0几点的降低
0. 首先overview一波:inference共有两处用到nms
第一次nms: proposal.py生成粗糙rois,从6000筛选出300个
第二次nms: test.py中,每张图,对每个类别,进行nms抑制,将rois中重叠度高的的rois抑制
测试发现,
- 只将第一次nms改为softnms效果几乎没有(人:0.09提点),但耗时大大增加(为原始的1.6倍)
- 分析原因是6000变为300个时进行soft导致循环比对耗时过长
- 提点不高是因为对于粗糙框进行soft,没啥意义
- 只将第二次nms改为softnms效果显著,且耗时几乎不增加(一丢丢)
- 分析提点原因是:同一类别的相互遮挡导致的nms误抑制–得到缓解
- 速度原因:只有小于300个rois进行softnms,速度几乎无影响
- 两者都改和只改第二次几乎一样,但耗时同样增加
- 提点效果=①+②
- 速度=①+②
- 结论:只改第二次的nms为softnms性价比高
1. 复制softnms_cpu_torch.py到lib/model/nms/
2. 改test.py
①先import
from model.nms.softnms_cpu_torch import softnms_cpu_torch
②305行左右:
cls_dets = cls_dets[order]
# keep = nms(cls_boxes[order, :], cls_scores[order], cfg.TEST.NMS)
keep = softnms_cpu_torch(cls_dets)
# cls_dets = cls_dets[keep.view(-1).long()]
cls_dets = keep
- 注意
- 原nms返回的keep是rois的索引,所以需要在cls_dets中再进行一波检索,
- 而softnms_cpu_torch返回的keep直接是roi的四点坐标+score值,因此需要把第三行注释掉
3. proposal.py
原版
# 6. apply nms (e.g. threshold = 0.7)
# 7. take after_nms_topN (e.g. 300)
# 8. return the top proposals (-> RoIs top)
keep_idx_i = nms(proposals_single, scores_single.squeeze(1), nms_thresh)
keep_idx_i = keep_idx_i.long().view(-1)
if post_nms_topN > 0:
keep_idx_i = keep_idx_i[:post_nms_topN]
proposals_single = proposals_single[keep_idx_i, :]
scores_single = scores_single[keep_idx_i, :]
# padding 0 at the end.
num_proposal = proposals_single.size(0)
output[i,:,0] = i
output[i,:num_proposal,1:] = proposals_single
return output
改后:
# 6. apply nms (e.g. threshold = 0.7)
# 7. take after_nms_topN (e.g. 300)
# 8. return the top proposals (-> RoIs top)
# keep_idx_i = nms(proposals_single, scores_single.squeeze(1), nms_thresh)
# keep_idx_i = keep_idx_i.long().view(-1)
det = torch.cat((proposals_single,scores_single),1)
keep_det = softnms_cpu_torch(det)
if post_nms_topN > 0:
# keep_idx_i = keep_idx_i[:post_nms_topN]
keep_det = keep_det[:post_nms_topN]
# proposals_single = proposals_single[keep_idx_i, :]
proposals_single = keep_det[:,:-1]
# scores_single = scores_single[keep_idx_i, :]
scores_single = keep_det[:,-1]
# padding 0 at the end.
num_proposal = proposals_single.size(0)
output[i,:,0] = i
output[i,:num_proposal,1:] = proposals_single
return output
附录 softnms_cpu_torch.py
注:用在test.py中,score_threshold=0.001
from __future__ import absolute_import
import numpy as np
import torch
import time
def area_of(left_top, right_bottom):
"""Compute the areas of rectangles given two corners.
Args:
left_top (N, 2): left top corner.
right_bottom (N, 2): right bottom corner.
Returns:
area (N): return the area.
return types: torch.Tensor
"""
hw = torch.clamp(right_bottom - left_top, min=0.0)
return hw[..., 0] * hw[..., 1]
def iou_of(boxes0, boxes1, eps=1e-5):
"""Return intersection-over-union (Jaccard index) of boxes.
Args:
boxes0 (N, 4): ground truth boxes.
boxes1 (N or 1, 4): predicted boxes.
eps: a small number to avoid 0 as denominator.
Returns:
iou (N): IoU values.
"""
overlap_left_top = torch.max(boxes0[..., :2], boxes1[..., :2])
overlap_right_bottom = torch.min(boxes0[..., 2:], boxes1[..., 2:])
overlap_area = area_of(overlap_left_top, overlap_right_bottom)
area0 = area_of(boxes0[..., :2], boxes0[..., 2:])
area1 = area_of(boxes1[..., :2], boxes1[..., 2:])
return overlap_area / (area0 + area1 - overlap_area + eps)
def softnms_cpu_torch(box_scores, score_threshold=0.001, sigma=0.5, top_k=-1):
"""Soft NMS implementation.
References:
https://arxiv.org/abs/1704.04503
https://github.com/facebookresearch/Detectron/blob/master/detectron/utils/cython_nms.pyx
Args:
box_scores (N, 5): boxes in corner-form and probabilities.
score_threshold: boxes with scores less than value are not considered.
sigma: the parameter in score re-computation.
scores[i] = scores[i] * exp(-(iou_i)^2 / simga)
top_k: keep top_k results. If k <= 0, keep all the results.
Returns:
picked_box_scores (K, 5): results of NMS.
"""
picked_box_scores = []
while box_scores.size(0) > 0:
max_score_index = torch.argmax(box_scores[:, 4])
cur_box_prob = box_scores[max_score_index, :].clone()
picked_box_scores.append(cur_box_prob)
if len(picked_box_scores) == top_k > 0 or box_scores.size(0) == 1:
break
cur_box = cur_box_prob[:-1]
box_scores[max_score_index, :] = box_scores[-1, :]
box_scores = box_scores[:-1, :]
ious = iou_of(cur_box.unsqueeze(0), box_scores[:, :-1])
box_scores[:, -1] = box_scores[:, -1] * torch.exp(-(ious * ious) / sigma)
box_scores = box_scores[box_scores[:, -1] > score_threshold, :]
if len(picked_box_scores) > 0:
return torch.stack(picked_box_scores)
else:
return torch.tensor([])
frcnn使用softnms方法二:跑softnms源码作者提供的cpu_nms.pyx(pytorch1.0下暂时没跑出来,但流程应该没错)
主要参考 https://blog.csdn.net/e01528/article/details/80800122 (注:这篇在解决报错的时候方法不对,参照我下面的setup.py就不会报错了)
- 下载源码cpu_nms.pyx放在和nms_wrapper一个文件夹下:
https://github.com/bharatsingh430/soft-nms/blob/master/lib/nms/cpu_nms.pyx
- 改nms_wrapper
from fast_rcnn.config import cfg
from nms.gpu_nms import gpu_nms
from nms.cpu_nms import cpu_nms, cpu_soft_nms
import numpy as np
def soft_nms(dets, sigma=0.5, Nt=0.3, threshold=0.001, method=1):
keep = cpu_soft_nms(np.ascontiguousarray(dets, dtype=np.float32),
np.float32(sigma), np.float32(Nt),
np.float32(threshold),
np.uint8(method))
return keep
# Original NMS implementation
def nms(dets, thresh, force_cpu=False):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if cfg.USE_GPU_NMS and not force_cpu:
return gpu_nms(dets, thresh, device_id=cfg.GPU_ID)
else:
return cpu_nms(dets, thresh)
- 创建setup.py文件:放在和cpu_nms.pyx同路径下,用来编译cpu_nms.pyx
from distutils.core import setup
from Cython.Build import cythonize
import numpy
setup(
name='softnms_module',
ext_modules=cythonize('cpu_nms.pyx'),
include_dirs=[numpy.get_include()]
)
- 编译
python setup.py build
如果编译遇到错误:Cython:“fatal error: numpy/arrayobject.h: No such file or directory”
是因为没有include numpy的路径(但按照我上面写的setup.py是不会报错的)
解决原理参考:https://cloud.tencent.com/developer/ask/73157
- 将生成的build文件夹下面的so文件,拷贝到主文件夹下面,删除build文件夹,即可。
- 在test.py中加入:
①先import和定义函数
from model.nms.cpu_nms import cpu_soft_nms
def soft_nms(dets, sigma=0.5, Nt=0.3, threshold=0.001, method=1):
keep = cpu_soft_nms(dets,
np.float32(sigma), np.float32(Nt),
np.float32(threshold),
np.uint8(method))
return keep
def psoft(cls_dets):
cls_dets_ny = cls_dets.numpy().astype(np.float32)
keep = soft_nms(box_scores = cls_dets_ny)
keep = torch.Tensor(keep).long()
return cls_dets[keep]
②第305行左右
cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1)
cls_dets = cls_dets[order]
# keep = nms(cls_boxes[order, :], cls_scores[order], cfg.TEST.NMS)
# cls_dets = cls_dets[keep.view(-1).long()]
cls_dets = psoft(cls_dets)
其他资料
nms的作用地方+nms的四种实现方法
http://www.cnblogs.com/king-lps/p/9031568.html
softnms解析
https://blog.csdn.net/lanyuxuan100/article/details/78767818
softnms源码
https://github.com/bharatsingh430/soft-nms/blob/master/lib/fast_rcnn/test.py
普通nms的GPU实现
https://blog.csdn.net/qq_21368481/article/details/85722590
todolist
- softnms.cu
- 跑通github原版softnms.pyx用Markdown编辑器