yolov8添加FPPI评价指标

续yolov7添加FPPI评价指标 。之前在yolov7中增加了fppi指标，有不少网友问有没有yolov8中增加，最近没有做算法训练，也一直没时间弄。这几天晚上抽了点时间，弄了一下。不过我本地的yolov8挺久没更新了，所以不一定能直接用，权当参考吧。

yolov8 中FPPI实现

yolo8中的评价指标实现位于ultralytics/utils/metrics.py中，我们只需要参照mAP指标在其中增加FPPI的内容即可：

def fppi_per_class(tp, conf, pred_cls, target_cls, image_num, plot=False, on_plot=None, save_dir=Path(), names=(), eps=1e-16, prefix=""):
    """Compute the false positives per image (FPPW) metric, given the recall and precision curves.
    Source:
    # Arguments
        tp:  True positives (nparray, nx1 or nx10).
        conf:  Objectness value from 0-1 (nparray).
        pred_cls:  Predicted object classes (nparray).
        target_cls:  True object classes (nparray).
        plot:  Plot precision-recall curve at mAP@0.5
        save_dir:  Plot save directory
    # Returns
        The fppi curve
    """
    # Sort by objectness
    i = np.argsort(-conf)
    tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]

    # Find unique classes
    unique_classes = np.unique(target_cls)
    nc = unique_classes.shape[0]  # number of classes, number of detections

    # Create Precision-Recall curve and compute AP for each class
    px, py = np.linspace(0, 1, 1000), np.linspace(0, 100, 1000)  # for plotting
    r = np.zeros((nc, 1000))
    miss_rate = np.zeros((nc, 1000))
    fppi = np.zeros((nc, 1000))
    miss_rate_at_fppi = np.zeros((nc, 3))  # missrate at fppi 1, 0.1, 0.01
    p_miss_rate = np.array([1, 0.1, 0.01])
    for ci, c in enumerate(unique_classes):
        i = pred_cls == c
        n_l = (target_cls == c).sum()  # number of labels
        n_p = i.sum()  # number of predictions

        if n_p == 0 or n_l == 0:
            continue
        else:
            # Accumulate FPs and TPs
            fpc = (1 - tp[i]).cumsum(0)
            tpc = tp[i].cumsum(0)

            # Recall
            recall = tpc / (n_l + eps)  # recall curve
            r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0)  # negative x, xp because xp decreases
            miss_rate[ci] = 1 - r[ci]

            fp_per_image = fpc / image_num
            fppi[ci] = np.interp(-px, -conf[i], fp_per_image[:, 0], left=0)

            miss_rate_at_fppi[ci] = np.interp(-p_miss_rate, -fppi[ci], miss_rate[ci])

    if plot:
        names = [v for k, v in names.items() if k in unique_classes]  # list: only classes that have data
        names = dict(enumerate(names)) 
        plot_fppi_curve(fppi, miss_rate, miss_rate_at_fppi, save_dir / f"{prefix}fppi_curve.png", names, on_plot=on_plot)


    return miss_rate, fppi, miss_rate_at_fppi

将fppi以对数坐标画图：

@plt_settings()
def plot_fppi_curve(px, py, fppi, save_dir=Path("pr_curve.png"), names=(), on_plot=None):
    """Plots a missrate-fppi curve."""
    fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)
    py = np.stack(py, axis=1)
    # semi log
    for i, y in enumerate(py.T):
        ax.semilogx(px[i],y, linewidth=1, label=f'{names[i]} {fppi[i].mean():.3f}')  # plot(recall, precision)
    
    ax.semilogx(px.mean(0), py.mean(1), linewidth=3, color='blue', label='all classes %.3f' % fppi.mean())

    ax.set_xlabel('False Positives Per Image')
    ax.set_ylabel('Miss Rate')
    ax.set_xlim(0, 100)
    ax.set_ylim(0, 1)
    ax.grid(True)
    plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left")
    fig.savefig(Path(save_dir), dpi=250)
    plt.close(fig)
    if on_plot:
        on_plot(save_dir)

至此，fppi相关实现就完成了，接下来就是在代码中调用了。

测试中调用

通过调试，metrics的调用是在ultralytics/models/yolo/detect/val.py中的DetectionValidator中的get_stats函数中，在调用get_stats之前，会调用update_metrics计算tp, fp等。首先我们需要解决的时fppi计算需要图片的张数这个参数。在update_metrics我们可以获取这个值，在init_metrics增加参数self.image_num=0,在update_metrics中更新：

def update_metrics(self, preds, batch):
        """Metrics."""
        self.image_num+=len(batch['im_file'])
        for si, pred in enumerate(preds):
            idx = batch['batch_idx'] == si
            cls = batch['cls'][idx]
            bbox = batch['bboxes'][idx]
            ...

ap的计算在ultralytics/utils/metrics.py的DetMetrics中，我们给它也增加一个self.image_num=0参数, 并且在get_stats函数中设置它：

def get_stats(self):
        """Returns metrics statistics and results dictionary."""
        # set image_num for fppi calculation
        self.metrics.image_num = self.image_num
        stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*self.stats)]  # to numpy
        if len(stats) and stats[0].any():
            self.metrics.process(*stats)
        self.nt_per_class = np.bincount(stats[-1].astype(int), minlength=self.nc)  # number of targets per class
        return self.metrics.results_dict

然后修改其process函数，在这个函数中调用乐ap_per_class来计算map, 我们在后面增加计算fppi的函数即可( def keys(self)中要加一个fppi的key)：

def process(self, tp, conf, pred_cls, target_cls):
        """Process predicted results for object detection and update metrics."""
        results = ap_per_class(
            tp,
            conf,
            pred_cls,
            target_cls,
            plot=self.plot,
            save_dir=self.save_dir,
            names=self.names,
            on_plot=self.on_plot,
        )[2:]
        self.box.nc = len(self.names)
        self.box.update(results)
        results_fppi = fppi_per_class(
            tp,
            conf,
            pred_cls,
            target_cls,
            self.image_num,
            plot=self.plot,
            save_dir=self.save_dir,
            names=self.names,
            on_plot=self.on_plot,
        )
        self.box.update_fppi(results_fppi)
        
 @property
    def keys(self):
        """Returns a list of keys for accessing specific metrics."""
        return ["metrics/precision(B)", "metrics/recall(B)", "metrics/mAP50(B)", "metrics/mAP50-95(B)",  "metrics/missrate_fppi_1"]

指标的计算结果是保存在ultralytics/utils/metrics.py的Metric结构中，所以还需要增加存储fppi相关指标(fitness也修改)：

class Metric(SimpleClass):
    """
    Class for computing evaluation metrics for YOLOv8 model.

    Attributes:
        p (list): Precision for each class. Shape: (nc,).
        r (list): Recall for each class. Shape: (nc,).
        f1 (list): F1 score for each class. Shape: (nc,).
        all_ap (list): AP scores for all classes and all IoU thresholds. Shape: (nc, 10).
        ap_class_index (list): Index of class for each AP score. Shape: (nc,).
        nc (int): Number of classes.

    Methods:
        ap50(): AP at IoU threshold of 0.5 for all classes. Returns: List of AP scores. Shape: (nc,) or [].
        ap(): AP at IoU thresholds from 0.5 to 0.95 for all classes. Returns: List of AP scores. Shape: (nc,) or [].
        mp(): Mean precision of all classes. Returns: Float.
        mr(): Mean recall of all classes. Returns: Float.
        map50(): Mean AP at IoU threshold of 0.5 for all classes. Returns: Float.
        map75(): Mean AP at IoU threshold of 0.75 for all classes. Returns: Float.
        map(): Mean AP at IoU thresholds from 0.5 to 0.95 for all classes. Returns: Float.
        mean_results(): Mean of results, returns mp, mr, map50, map.
        class_result(i): Class-aware result, returns p[i], r[i], ap50[i], ap[i].
        maps(): mAP of each class. Returns: Array of mAP scores, shape: (nc,).
        fitness(): Model fitness as a weighted combination of metrics. Returns: Float.
        update(results): Update metric attributes with new evaluation results.

    """

    def __init__(self) -> None:
        self.p = []  # (nc, )
        self.r = []  # (nc, )
        self.f1 = []  # (nc, )
        self.all_ap = []  # (nc, 10)
        self.ap_class_index = []  # (nc, )
        self.nc = 0
        self.missrate = [] #miss rate, (nc, )
        self.fppi = [] #false positives per image, (nc, )
        self.missrate_fppi = [] #miss rate at fppi 1, 0.1, 0.01, (nc, 3)

    @property
    def missrate_fppi_1(self):
        return self.missrate_fppi[:, 0].mean() if len(self.missrate_fppi) else []

    @property
    def ap50(self):
        """
        Returns the Average Precision (AP) at an IoU threshold of 0.5 for all classes.

        Returns:
            (np.ndarray, list): Array of shape (nc,) with AP50 values per class, or an empty list if not available.
        """
        return self.all_ap[:, 0] if len(self.all_ap) else []

    @property
    def ap(self):
        """
        Returns the Average Precision (AP) at an IoU threshold of 0.5-0.95 for all classes.

        Returns:
            (np.ndarray, list): Array of shape (nc,) with AP50-95 values per class, or an empty list if not available.
        """
        return self.all_ap.mean(1) if len(self.all_ap) else []

    @property
    def mp(self):
        """
        Returns the Mean Precision of all classes.

        Returns:
            (float): The mean precision of all classes.
        """
        return self.p.mean() if len(self.p) else 0.0

    @property
    def mr(self):
        """
        Returns the Mean Recall of all classes.

        Returns:
            (float): The mean recall of all classes.
        """
        return self.r.mean() if len(self.r) else 0.0

    @property
    def map50(self):
        """
        Returns the mean Average Precision (mAP) at an IoU threshold of 0.5.

        Returns:
            (float): The mAP50 at an IoU threshold of 0.5.
        """
        return self.all_ap[:, 0].mean() if len(self.all_ap) else 0.0

    @property
    def map75(self):
        """
        Returns the mean Average Precision (mAP) at an IoU threshold of 0.75.

        Returns:
            (float): The mAP50 at an IoU threshold of 0.75.
        """
        return self.all_ap[:, 5].mean() if len(self.all_ap) else 0.0

    @property
    def map(self):
        """
        Returns the mean Average Precision (mAP) over IoU thresholds of 0.5 - 0.95 in steps of 0.05.

        Returns:
            (float): The mAP over IoU thresholds of 0.5 - 0.95 in steps of 0.05.
        """
        return self.all_ap.mean() if len(self.all_ap) else 0.0

    def mean_results(self):
        """Mean of results, return mp, mr, map50, map."""
        return [self.mp, self.mr, self.map50, self.map, self.missrate_fppi_1]

    def class_result(self, i):
        """class-aware result, return p[i], r[i], ap50[i], ap[i]."""
        return self.p[i], self.r[i], self.ap50[i], self.ap[i]

    @property
    def maps(self):
        """mAP of each class."""
        maps = np.zeros(self.nc) + self.map
        for i, c in enumerate(self.ap_class_index):
            maps[c] = self.ap[i]
        return maps

    def fitness(self):
        """Model fitness as a weighted combination of metrics."""
        w = [0.0, 0.0, 0.1, 0.9]  # weights for [P, R, mAP@0.5, mAP@0.5:0.95]
        return (np.array(self.mean_results()[:4]) * w).sum()

    def update(self, results):
        """
        Args:
            results (tuple): A tuple of (p, r, ap, f1, ap_class)
        """
        self.p, self.r, self.f1, self.all_ap, self.ap_class_index = results
    
    def update_fppi(self, results):
        """
        Args:
            results (tuple): A tuple of (missrate, fppi, missrate_fppi)
        """
        self.missrate, self.fppi, self.missrate_fppi = results

效果

训练过程中会在mAP的后面打印fppi, 训练完成后以及调用test.py测试时，会画fppi图：

结语

本文简述了在yolov8中增加FPPI评价指标，可以用来直观的表现模型的效果，指导阈值的选取。