目标检测中使用二部图匹配算法

在SSD的原文中有一段话：

Matching strategy During training we need to determine which default boxes correspond to a ground truth detection and train the network accordingly. For each ground truth box we are selecting from default boxes that vary over location, aspect ratio, and scale. We begin by matching each ground truth box to the default box with the best jaccard overlap (as in MultiBox [7]). Unlike MultiBox, we then match default boxes to any ground truth with jaccard overlap higher than a threshold (0.5). This simplifies the learning problem, allowing the network to predict high scores for multiple overlapping default boxes rather than requiring it to pick only the one with maximum overlap.

SSD中进行匹配的思路是：

1）建图。对于某个预测框，遍历所有的gt_bbox，如果它们的交集大于0，那么就用一条边把gt_bbox与预测框连接起来；这样对所有的预测框都进行同样操作。如果把所有gt_bbox放入集合A，所有预测框放入集合B，那么这一步就是建立A与B之间的二部图，二部图中边的权值为预测框与gt_bbox的iou分数。

2）匹配。匹配分为2个阶段。

第一个阶段：找出二部图中的边权值最大的边，并该边对应的gt_bbox与预测框从顶点集中删除；反复进行这个过程，直到所有的gt_bbox都找到匹配的预测框；

第二个阶段：如果匹配的类型是BIPARTITE，那么匹配过程已结束；如果匹配类型是PER_PREDICTION，表示对于每个预测框，都要找到一个gt_bbox与之匹配，那么对于第一阶段未匹配上的预测框，从gt_bboxes集合中找到与它所连边中权值最大的gt_bbox作为它的匹配。

第一个阶段的匹配保证每个gt_bbox都至少有一个预测框与之匹配，这是因为预测框足够多，必定有许多预测框与gt_bbox建立边连接，且每个预测框都有0个或1个gt_bbox与之匹配。

第二个阶段的匹配如果采用PER_PREDICTION方法，那么每个预测框都有且只有1个gt_bbox与之匹配了，但是gt_bbox可能匹配上多个预测框。

本来是想找到SSD中的二部图匹配方法与经典的匈牙利方法、KM算法或GS算法之间的联系的，但是与这些经典算法都不相同。

 

//from code:caffe-ssd/src/caffe/util/utils.cpp

void MatchBBox(const vector<NormalizedBBox>& gt_bboxes,
    const vector<NormalizedBBox>& pred_bboxes, const int label,
    const MatchType match_type, const float overlap_threshold,
    const bool ignore_cross_boundary_bbox,
    vector<int>* match_indices, vector<float>* match_overlaps) {
  int num_pred = pred_bboxes.size();
  match_indices->clear();
  match_indices->resize(num_pred, -1);
  match_overlaps->clear();
  match_overlaps->resize(num_pred, 0.);

  int num_gt = 0;
  vector<int> gt_indices;
  if (label == -1) {
    // label -1 means comparing against all ground truth.
    num_gt = gt_bboxes.size();
    for (int i = 0; i < num_gt; ++i) {
      gt_indices.push_back(i);
    }
  } else {
    // Count number of ground truth boxes which has the desired label.
    for (int i = 0; i < gt_bboxes.size(); ++i) {
      if (gt_bboxes[i].label() == label) {
        num_gt++;
        gt_indices.push_back(i);
      }
    }
  }
  if (num_gt == 0) {
    return;
  }

  // Store the positive overlap between predictions and ground truth.
  map<int, map<int, float> > overlaps;//map的第1个key对应default_bbox，第2个key对应gt_bbox
  //LOG(INFO)<<"num_pred="<<num_pred;//1917
  for (int i = 0; i < num_pred; ++i) {
    if (ignore_cross_boundary_bbox && IsCrossBoundaryBBox(pred_bboxes[i])) {
      (*match_indices)[i] = -2;
      continue;
    }
    for (int j = 0; j < num_gt; ++j) {
      float overlap = JaccardOverlap(pred_bboxes[i], gt_bboxes[gt_indices[j]]);
       //对每个预测框，找到交集最大gt_bbox
      if (overlap > 1e-6) {
        (*match_overlaps)[i] = std::max((*match_overlaps)[i], overlap);
        overlaps[i][j] = overlap;
      }
    }
  }

  // Bipartite matching.
  vector<int> gt_pool;
  for (int i = 0; i < num_gt; ++i) {
    gt_pool.push_back(i);
  }
  //LOG(INFO)<<"gt_pool.size="<<gt_pool.size();
  //要使得每个gt_bbox都要有一个default_bbox与之匹配上
  while (gt_pool.size() > 0) {
    // Find the most overlapped gt and cooresponding predictions.
    int max_idx = -1;
    int max_gt_idx = -1;
    float max_overlap = -1;
    //遍历之前找到的所有重叠框（此配对可能是多个default_bbox对应一个gt_bbox）
    //找出交集最大的重叠框，并把其gt_bbox移除gt_pool,default_bbox标记为已配对
    for (map<int, map<int, float> >::iterator it = overlaps.begin();
         it != overlaps.end(); ++it) {
      int i = it->first;//the i-th default bboxes
      if ((*match_indices)[i] != -1) {
        // The prediction already has matched ground truth or is ignored.
        continue;
      }
      for (int p = 0; p < gt_pool.size(); ++p) {//match each gt_bbox to default bboxes.
        int j = gt_pool[p];
        if (it->second.find(j) == it->second.end()) {
          // No overlap between the i-th prediction and j-th ground truth.
          continue;
        }
        // Find the maximum overlapped pair.
        if (it->second[j] > max_overlap) {
          // If the prediction has not been matched to any ground truth,
          // and the overlap is larger than maximum overlap, update.
          max_idx = i;
          max_gt_idx = j;
          max_overlap = it->second[j];
        }
      }
    }
    if (max_idx == -1) {
      // Cannot find good match.
      break;
    } else {
      CHECK_EQ((*match_indices)[max_idx], -1);
      (*match_indices)[max_idx] = gt_indices[max_gt_idx];
      (*match_overlaps)[max_idx] = max_overlap;
      // Erase the ground truth.
      gt_pool.erase(std::find(gt_pool.begin(), gt_pool.end(), max_gt_idx));
    }
  }

  switch (match_type) {
    case MultiBoxLossParameter_MatchType_BIPARTITE:
      // Already done.
      break;
    case MultiBoxLossParameter_MatchType_PER_PREDICTION:
      // Get most overlaped for the rest prediction bboxes.
      for (map<int, map<int, float> >::iterator it = overlaps.begin();
           it != overlaps.end(); ++it) {
        int i = it->first;
        if ((*match_indices)[i] != -1) {
          // The prediction already has matched ground truth or is ignored.
          continue;
        }
        int max_gt_idx = -1;
        float max_overlap = -1;
        for (int j = 0; j < num_gt; ++j) {
          if (it->second.find(j) == it->second.end()) {
            // No overlap between the i-th prediction and j-th ground truth.
            continue;
          }
          // Find the maximum overlapped pair.
          float overlap = it->second[j];//the IoU pred_i and gt_j
          if (overlap >= overlap_threshold && overlap > max_overlap) {
            // If the prediction has not been matched to any ground truth,
            // and the overlap is larger than maximum overlap, update.
            max_gt_idx = j;
            max_overlap = overlap;
          }
        }
        //LOG(INFO)<<"max_overlap="<<max_overlap;
        if (max_gt_idx != -1) {
          // Found a matched ground truth.
          CHECK_EQ((*match_indices)[i], -1);
          (*match_indices)[i] = gt_indices[max_gt_idx];//the best gt's label is assigned to pred_i 
          (*match_overlaps)[i] = max_overlap;
        }
      }
      break;
    default:
      LOG(FATAL) << "Unknown matching type.";
      break;
  }

  return;
}