PV-RCNN代码解读——eval.py

PV-RCNN：paper，code

（一）目标检测模型评估的简介

训练完数据并且测试最后一个epoch之后，log的结果

INFO  Car AP@0.70, 0.70, 0.70:
bbox AP:89.6559, 83.1009, 78.4558
bev  AP:87.8147, 77.5440, 76.1072
3d   AP:75.0818, 64.6598, 58.0468
aos  AP:87.94, 80.87, 75.62

这里涉及到目标检测中几个重要的定义：

IoU（Intersection over union）：交并比IoU衡量的是两个区域的重叠程度，是两个区域重叠部分面积占二者总面积的比例。在目标检测中，如果模型输出的结果与真值gt的交并比 > 某个阈值（0.5或0.7）时，即认为我们的模型输出了正确的结果。

Precision ：检索出来的条目中有多大比例是我们需要的。

Recall：我们需要的条目中有多大比例被检索出来了。

AP(Average Precision)：平均精准度，对Precision-Recall曲线上的Precision值求均值。

（二）evaluation函数

当你运行test.py进行测试，会用到evaluation函数

	# tools/test.py
    result_str, result_dict = dataset.evaluation(
        det_annos, class_names,
        eval_metric=cfg.MODEL.POST_PROCESSING.EVAL_METRIC,
        output_path=final_output_dir
    )

evaluation函数的定义

    # pcdet/datasets/kitti/kitti_dataset.py 
    def evaluation(self, det_annos, class_names, **kwargs):
        from .kitti_object_eval_python import eval as kitti_eval
        
        eval_det_annos = copy.deepcopy(det_annos)
        eval_gt_annos = [copy.deepcopy(info['annos']) for info in self.kitti_infos]
        ap_result_str, ap_dict = kitti_eval.get_official_eval_result(eval_gt_annos, eval_det_annos, class_names)
        
        return ap_result_str, ap_dict

get_official_eval_result函数的定义

# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
# 打印测试结果
def get_official_eval_result(gt_annos, dt_annos, current_classes, PR_detail_dict=None):
    
    overlap_0_7 = np.array([[0.7, 0.5, 0.5, 0.7, 0.5, 0.7],
                            [0.7, 0.5, 0.5, 0.7, 0.5, 0.7],
                            [0.7, 0.5, 0.5, 0.7, 0.5, 0.7]])
    overlap_0_5 = np.array([[0.7, 0.5,  0.5,  0.7,  0.5, 0.5], 
    						[0.5, 0.25, 0.25, 0.5, 0.25, 0.5],
                            [0.5, 0.25, 0.25, 0.5, 0.25, 0.5]])
   
    # 目的就是给不同的类别设置不同的阈值
    # 每个类别输出2*2个结果，AP或AP_R40，overlap=0.7或0.5
    min_overlaps = np.stack([overlap_0_7, overlap_0_5], axis=0)  # [2, 3, 5]
    class_to_name = {
        0: 'Car', 			#[[0.7,0.7,0.7],[0.5, 0.5, 0.5]]
        1: 'Pedestrian',	#[[0.5,0.5,0.5],[0.5,0.25,0.25]]
        2: 'Cyclist',		#[[0.5,0.5,0.5],[0.5,0.25,0.25]]
        3: 'Van',			#[[0.7,0.7,0.7],[0.7, 0.5, 0.5]]
        4: 'Person_sitting',#[[0.5,0.5,0.5],[0.5,0.25,0.25]]
        5: 'Truck'			#[[0.7,0.7,0.7],[0.5, 0.5, 0.5]]
    }
    
    name_to_class = {v: n for n, v in class_to_name.items()}
    min_overlaps = min_overlaps[:, :, current_classes]
    result = ''
	
	# 计算结果的函数do_eval()（后面详细说）
    mAPbbox, mAPbev, mAP3d, mAPaos, mAPbbox_R40, mAPbev_R40, mAP3d_R40, mAPaos_R40 = do_eval(
        gt_annos, dt_annos, current_classes, min_overlaps, compute_aos, PR_detail_dict=PR_detail_dict)

	#打印结果
    ret_dict = {}
    for j, curcls in enumerate(current_classes): #每一种类
        # mAP threshold array: [num_minoverlap, metric, class]
        # mAP result: [num_class, num_diff, num_minoverlap]
        for i in range(min_overlaps.shape[0]): # 2
            result += print_str(
                (f"{class_to_name[curcls]} "
                 "AP@{:.2f}, {:.2f}, {:.2f}:".format(*min_overlaps[i, :, j])))
            result += print_str((f"bbox AP:{mAPbbox[j, 0, i]:.4f}, "
                                 f"{mAPbbox[j, 1, i]:.4f}, "
                                 f"{mAPbbox[j, 2, i]:.4f}"))
            result += print_str((f"bev  AP:{mAPbev[j, 0, i]:.4f}, "
                                 f"{mAPbev[j, 1, i]:.4f}, "
                                 f"{mAPbev[j, 2, i]:.4f}"))
            result += print_str((f"3d   AP:{mAP3d[j, 0, i]:.4f}, "
                                 f"{mAP3d[j, 1, i]:.4f}, "
                                 f"{mAP3d[j, 2, i]:.4f}"))
            if compute_aos:
                result += print_str((f"aos  AP:{mAPaos[j, 0, i]:.2f}, "
                                     f"{mAPaos[j, 1, i]:.2f}, "
                                     f"{mAPaos[j, 2, i]:.2f}"))

            '''省略了把AP换成AP_R40的部分，因为代码是类似的'''
    return result, ret_dict

用到的几个函数定义如下

# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
def print_str(value, *arg, sstream=None): #打印结果的函数
    if sstream is None:
        sstream = sysio.StringIO()
    sstream.truncate(0)
    sstream.seek(0)
    print(value, *arg, file=sstream)
    return sstream.getvalue()

（三）do_eval函数

do_eval是计算评估结果的重要函数，其定义如下

# 计算评估结果
# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
def do_eval(gt_annos, dt_annos, current_classes, min_overlaps, compute_aos=False, PR_detail_dict=None):
    
    # min_overlaps: [num_minoverlap, metric, num_class]
    difficultys = [0, 1, 2]
    
	'''这里只保留了计算3d的代码，bev,aos,bbox是类似的'''
    ret = eval_class(gt_annos, dt_annos, current_classes, difficultys, 2, min_overlaps)
    # ret: [num_class, num_diff, num_minoverlap, num_sample_points]
    
    mAP_3d = get_mAP(ret["precision"])
    mAP_3d_R40 = get_mAP_R40(ret["precision"])
    if PR_detail_dict is not None:
        PR_detail_dict['3d'] = ret['precision']
    return mAP_bbox, mAP_bev, mAP_3d, mAP_aos, mAP_bbox_R40, mAP_bev_R40, mAP_3d_R40, mAP_aos_R40

get_mAP和get_mAP_R40函数的定义

# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
def get_mAP(prec): # 计算mAP
    sums = 0
    for i in range(0, prec.shape[-1], 4):
        sums = sums + prec[..., i]
    return sums / 11 * 100

def get_mAP_R40(prec): #计算mAP_R40
    sums = 0
    for i in range(1, prec.shape[-1]):
        sums = sums + prec[..., i]
    return sums / 40 * 100

eval_class的定义

# pcdet/datasets/kitti/kitti_object_eval_python/eval.py
def eval_class(gt_annos,
               dt_annos,
               current_classes,
               difficultys,  # tuple类型， (0, 1, 2)
               metric,       # 0 (bbox), 1 (bev)或 2 (3d)
               min_overlaps, # (2, 3, num_classes) 其中:
                             # 2 表示阈值为中等或者容易
                             # 3 表示表示不同的指标 (bbox, bev, 3d), 
                             # num_classes用于每个类的阈值
               compute_aos=False,
               num_parts=100):
    """这里是官方注释的各个参量的意义
    Kitti eval. support 2d/bev/3d/aos eval. support 0.5:0.05:0.95 coco AP.
    Args:
        gt_annos: dict, must from get_label_annos() in kitti_common.py
        dt_annos: dict, must from get_label_annos() in kitti_common.py
        current_classes: list of int, 0: car, 1: pedestrian, 2: cyclist
        difficultys: list of int. eval difficulty, 0: easy, 1: normal, 2: hard
        metric: eval type. 0: bbox, 1: bev, 2: 3d
        min_overlaps: float, min overlap. format: [num_overlap, metric, class].
        num_parts: int. a parameter for fast calculate algorithm

    Returns:
        dict of recall, precision and aos
    """
    assert len(gt_annos) == len(dt_annos)
    num_examples = len(gt_annos)
    split_parts = get_split_parts(num_examples, num_parts)
	
	# 计算iou，calculate_iou_partly函数解读代码参见文末链接
    rets = calculate_iou_partly(dt_annos, gt_annos, metric, num_parts)
    (overlaps, parted_overlaps, total_dt_num, total_gt_num) = rets
    
    N_SAMPLE_PTS = 41
    
    num_minoverlap = len(min_overlaps)
    num_class = len(current_classes)
    num_difficulty = len(difficultys)
    
    # 初始化precision,recall,aos
    precision = np.zeros([num_class, num_difficulty, num_minoverlap, N_SAMPLE_PTS])
    recall = np.zeros([num_class, num_difficulty, num_minoverlap, N_SAMPLE_PTS])
    aos = np.zeros([num_class, num_difficulty, num_minoverlap, N_SAMPLE_PTS])
    
    # 每个类别
    for m, current_class in enumerate(current_classes):
    	# 每个难度
        for l, difficulty in enumerate(difficultys):
        	# _prepare_data函数解读代码参见文末链接
            rets = _prepare_data(gt_annos, dt_annos, current_class, difficulty)
            (gt_datas_list, dt_datas_list, ignored_gts, ignored_dets,
             dontcares, total_dc_num, total_num_valid_gt) = rets
            
            # 运行两次，首先进行中等难度的总体设置，然后进行简单设置。
            for k, min_overlap in enumerate(min_overlaps[:, metric, m]):
                thresholdss = []

                # 循环浏览数据集中的图像。因此一次只显示一张图片。
                for i in range(len(gt_annos)):
                
                	# compute_statistics_jit函数解读代码参见文末链接
                    rets = compute_statistics_jit(
                        overlaps[i], # 单个图像的iou值b/n gt和dt
                        gt_datas_list[i], # N x 5阵列
                        dt_datas_list[i], # N x 6阵列
                        ignored_gts[i], # 长度N数组，-1、0、1
                        ignored_dets[i], # 长度N数组，-1、0、1
                        dontcares[i], # 无关框数量x 4
                        metric, # 0, 1, 或 2 (bbox, bev, 3d)
                        min_overlap=min_overlap, # 浮动最小IOU阈值为正
                        thresh=0.0, # 忽略得分低于此值的dt。
                        compute_fp=False)
                    tp, fp, fn, similarity, thresholds, _ = rets 
                    thresholdss += thresholds.tolist()
                    
                # 一维数组，记录匹配的dts分数
                thresholdss = np.array(thresholdss)
                thresholds = get_thresholds(thresholdss, total_num_valid_gt)
                thresholds = np.array(thresholds)
                # N_SAMPLE_PTS长度的一维数组，记录分数，递减，表示阈值

                # 储存有关gt/dt框的信息（是否忽略，fn，tn，fp）                
                pr = np.zeros([len(thresholds), 4])
                
                idx = 0
				# 我们将数据集分成多个部分并运行。
                for j, num_part in enumerate(split_parts):
                    gt_datas_part = np.concatenate(gt_datas_list[idx:idx + num_part], 0)
                    dt_datas_part = np.concatenate(dt_datas_list[idx:idx + num_part], 0)
                    dc_datas_part = np.concatenate(dontcares[idx:idx + num_part], 0)
                    ignored_dets_part = np.concatenate(ignored_dets[idx:idx + num_part], 0)
                    ignored_gts_part = np.concatenate(ignored_gts[idx:idx + num_part], 0)
                    # 再将各部分数据融合
                    fused_compute_statistics(parted_overlaps[j], pr, total_gt_num[idx:idx + num_part], total_dt_num[idx:idx + num_part], total_dc_num[idx:idx + num_part], gt_datas_part, dt_datas_part, dc_datas_part, ignored_gts_part, ignored_dets_part, metric, min_overlap=min_overlap,  thresholds=thresholds, compute_aos=compute_aos)
                    idx += num_part
                    
                for i in range(len(thresholds)): #计算recall和precision
                    recall[m, l, k, i] = pr[i, 0] / (pr[i, 0] + pr[i, 2]) #! true pos / (true pos + false neg)
                    precision[m, l, k, i] = pr[i, 0] / (pr[i, 0] + pr[i, 1]) # true pos / (true pos + false pos)
                    if compute_aos:
                        aos[m, l, k, i] = pr[i, 3] / (pr[i, 0] + pr[i, 1])
                
                # 返回各自序列的最值
                for i in range(len(thresholds)):
                    precision[m, l, k, i] = np.max(precision[m, l, k, i:], axis=-1)
                    recall[m, l, k, i] = np.max(recall[m, l, k, i:], axis=-1)
                    if compute_aos:
                        aos[m, l, k, i] = np.max(aos[m, l, k, i:], axis=-1)
    
    ret_dict = {
        "recall": recall, # [num_class, num_difficulty, num_minoverlap, N_SAMPLE_PTS]
        "precision": precision,  # RECALLING RECALL的顺序，因此精度降低
        "orientation": aos,
    }
    
    return ret_dict

致谢：detection-toolbox的作者将代码中的注释写的太清晰了，对我理解代码起到了很大帮助。有需要的读者可以看英文原版。

我的其他PV-RCNN代码解读系列文章，如果对你有帮助的话，请给我点赞哦~
PV-RCNN代码解读——输入参数介绍
PV-RCNN代码解读——TP，FP，TN，FN的计算
PV-RCNN代码解读——eval.py
PV-RCNN代码解读——计算iou
PV-RCNN代码解读——数据初始化
PV-RCNN代码解读——从点云到输入神经网络的数据处理