Faster RCNN minibatch.py

minibatch.py 的功能是： Compute minibatch blobs for training a Fast R-CNN network. 与roidb不同的是， minibatch中存储的并不是完整的整张图像图像，而是从图像经过转换后得到的四维blob以及从图像中截取的proposals，以及与之对应的labels等

def get_minibatch(roidb, num_classes) 该函数的功能是“Given a roidb, construct a minibatch sampled from it” ， 即从roidb中构造出一个minibatch来用于训练

def get_minibatch(roidb, num_classes):
    """Given a roidb, construct a minibatch sampled from it."""
    # 给定一个roidb，这个roidb中存储的可能是多张图片，也可能是单张或者多张图片，这个视cfg中的IMS_PER_BATCH 而定。比如在layer.py的 def _get_next_minibatch函数中，调用get_minibatch(minibatch_db, self._num_classes)中的 minibatch_db 只包含 IMS_PER_BATCH 张图片
    num_images = len(roidb)

    # Sample random scales to use for each image in this batch 给roidb中的图像随机分配缩放比例，从而加入图像的多尺度信息
    random_scale_inds = npr.randint(0, high=len(cfg.TRAIN.SCALES),
                                    size=num_images)
    assert(cfg.TRAIN.BATCH_SIZE % num_images == 0), \
        'num_images ({}) must divide BATCH_SIZE ({})'. \
        format(num_images, cfg.TRAIN.BATCH_SIZE)
    rois_per_image = cfg.TRAIN.BATCH_SIZE / num_images
    fg_rois_per_image = np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)

    # Get the input image blob, **formatted for caffe**
    # 将给定的roidb经过预处理（resize以及resize的scale），然后再利用im_list_to_blob函数来将图像转换成caffe支持的数据结构，即 N * C * H * W的四维结构
    im_blob, im_scales = _get_image_blob(roidb, random_scale_inds)

    # 最终的返回，字典。其中最主要的部分是‘data’,对应将roidb转换后得到的im_blob，其他的根据各个阶段，还会加入‘gt_boxes’、‘im_info’、‘rois’、‘labels’、‘bbox_targets’、‘bbox_inside_weights’、‘bbox_outside_weights’的信息
    blobs = {'data': im_blob}

    if cfg.TRAIN.HAS_RPN:
        assert len(im_scales) == 1, "Single batch only"
        assert len(roidb) == 1, "Single batch only"
        # gt boxes: (x1, y1, x2, y2, cls)
        gt_inds = np.where(roidb[0]['gt_classes'] != 0)[0]
        gt_boxes = np.empty((len(gt_inds), 5), dtype=np.float32)
        gt_boxes[:, 0:4] = roidb[0]['boxes'][gt_inds, :] * im_scales[0]
        gt_boxes[:, 4] = roidb[0]['gt_classes'][gt_inds]
        blobs['gt_boxes'] = gt_boxes
        # im_info表示h w scale这些信息
        blobs['im_info'] = np.array(
            [[im_blob.shape[2], im_blob.shape[3], im_scales[0]]],
            dtype=np.float32)
    else: # not using RPN
        # Now, build the region of interest and label blobs
        # labels_blob 和 rois_blob的shape都没有声明，后面会利用hstack和vstack将元素添加进去
        rois_blob = np.zeros((0, 5), dtype=np.float32)
        labels_blob = np.zeros((0), dtype=np.float32)

        bbox_targets_blob = np.zeros((0, 4 * num_classes), dtype=np.float32)
        bbox_inside_blob = np.zeros(bbox_targets_blob.shape, dtype=np.float32)
        # all_overlaps = []
        for im_i in xrange(num_images):
            # 遍历给定的roidb中的每张图片，随机组合sample of RoIs， 来生成前景样本和背景样本。
            # 返回包括每张图片中的roi（proposal）的坐标，所属的类别，bbox回归目标，bbox的inside_weight等
            labels, overlaps, im_rois, bbox_targets, bbox_inside_weights \
                = _sample_rois(roidb[im_i], fg_rois_per_image, rois_per_image,
                               num_classes)

            # Add to RoIs blob
            # _sample_rois返回的im_rois并没有缩放，所以这里要先缩放
            rois = _project_im_rois(im_rois, im_scales[im_i])

            batch_ind = im_i * np.ones((rois.shape[0], 1))
            rois_blob_this_image = np.hstack((batch_ind, rois))#给rois添加“所属哪张图片”信息
            # rois_blob存储的是这个roidb中每张图像的所有roi（proposals），每个proposals的存储信息包括其所属于哪张图片，以及roi的坐标
            rois_blob = np.vstack((rois_blob, rois_blob_this_image))
            # Add to labels, bbox targets, and bbox loss blobs
            # 将每张图像中rois的labels水平排列，得到的还是一个一维数组
            labels_blob = np.hstack((labels_blob, labels))

            bbox_targets_blob = np.vstack((bbox_targets_blob, bbox_targets))
            bbox_inside_blob = np.vstack((bbox_inside_blob, bbox_inside_weights))
            # all_overlaps = np.hstack((all_overlaps, overlaps))

        # For debug visualizations
        # _vis_minibatch(im_blob, rois_blob, labels_blob, all_overlaps)

        blobs['rois'] = rois_blob
        blobs['labels'] = labels_blob

        if cfg.TRAIN.BBOX_REG:
            blobs['bbox_targets'] = bbox_targets_blob
            blobs['bbox_inside_weights'] = bbox_inside_blob
            blobs['bbox_outside_weights'] = \
                np.array(bbox_inside_blob > 0).astype(np.float32)

    return blobs

---

def _get_image_blob(roidb, scale_inds)
该函数返回的是多张图像的blob 和 多张图像各自对应的scale
minibatch.py 中的_get_image_blob函数与generate.py 中的_get_image_blob 函数功能一样，只不过处理的是单张图像还是多张图像的问题。

def _get_image_blob(roidb, scale_inds):
    """Builds an input blob from the images in the roidb at the specified
    scales.
    """
    num_images = len(roidb)
    processed_ims = []
    im_scales = []
    for i in xrange(num_images):
        im = cv2.imread(roidb[i]['image'])

        #如果当前处理的是flipped的图像，那么，需要将图像所有的像素翻转，因为之前在准备imdb的时候，只是将
        #proposals，boxes的坐标翻转
        if roidb[i]['flipped']:
            im = im[:, ::-1, :]

        # prep_im_for_blob 函数的功能是获取经过resize的图像以及缩放的比例
        im, im_scale = prep_im_for_blob(im, cfg.PIXEL_MEANS, target_size,
                                        cfg.TRAIN.MAX_SIZE)
        im_scales.append(im_scale)
        processed_ims.append(im)

    # Create a blob to hold the input images 调用im_list_to_blob来将经过预处理的processed_ims转换成caffe支持的数据结构，即 N * C * H * W的四维结构
    blob = im_list_to_blob(processed_ims)

    return blob, im_scales

---

def _sample_rois(roidb, fg_rois_per_image, rois_per_image, num_classes)
在 def get_minibatch(roidb, num_classes) 中调用此函数，传进来的实参为单张图像的roidb ，该函数主要功能是随机组合sample of RoIs， 来生成前景样本和背景样本。

def _sample_rois(roidb, fg_rois_per_image, rois_per_image, num_classes):
    """Generate a random sample of RoIs comprising foreground and background
    examples.
    """
    # label = class RoI has max overlap with
    labels = roidb['max_classes']
    overlaps = roidb['max_overlaps']
    rois = roidb['boxes']

    # Select foreground RoIs as those with >= FG_THRESH overlap
    fg_inds = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0]
    # Guard against the case when an image has fewer than fg_rois_per_image
    # foreground RoIs
    fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
    # Sample foreground regions without replacement
    if fg_inds.size > 0:
        fg_inds = npr.choice(
                fg_inds, size=fg_rois_per_this_image, replace=False)

    # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
    bg_inds = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) &
                       (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
    # Compute number of background RoIs to take from this image (guarding
    # against there being fewer than desired)
    bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
    bg_rois_per_this_image = np.minimum(bg_rois_per_this_image,
                                        bg_inds.size)
    # Sample foreground regions without replacement
    if bg_inds.size > 0:
        bg_inds = npr.choice(
                bg_inds, size=bg_rois_per_this_image, replace=False)

    # The indices that we're selecting (both fg and bg)
    keep_inds = np.append(fg_inds, bg_inds)
    # Select sampled values from various arrays:
    labels = labels[keep_inds]
    # Clamp labels for the background RoIs to 0
    labels[fg_rois_per_this_image:] = 0
    overlaps = overlaps[keep_inds]
    rois = rois[keep_inds]

    # 调用_get_bbox_regression_labels函数，生成bbox_targets 和 bbox_inside_weights，
    #它们都是N * 4K 的ndarray，N表示keep_inds的size，也就是minibatch中样本的个数；bbox_inside_weights 
    #也随之生成
    bbox_targets, bbox_inside_weights = _get_bbox_regression_labels(
            roidb['bbox_targets'][keep_inds, :], num_classes)

    return labels, overlaps, rois, bbox_targets, bbox_inside_weights

---

def _get_bbox_regression_labels(bbox_target_data, num_classes):
该函数主要是获取bbox_target_data中回归目标的的4个坐标编码作为bbox_targets，同时生成bbox_inside_weights，它们都是N * 4K 的ndarray，N表示keep_inds的size，也就是minibatch中样本的个数。
该函数在def _sample_rois 中得以调用

def _get_bbox_regression_labels(bbox_target_data, num_classes):
    """Bounding-box regression targets are stored in a compact form in the
    roidb.

    This function expands those targets into the 4-of-4*K representation used
    by the network (i.e. only one class has non-zero targets). The loss weights
    are similarly expanded. 将非背景类的每个targets从（4，）拓展为（4K，），而其中只有对应类的
    bbox_targets才为非0

    Returns:
        bbox_targets (ndarray): N x 4K blob of regression targets
        bbox_inside_weights (ndarray): N x 4K blob of loss weights
    """
    clss = bbox_target_data[:, 0]
    bbox_targets = np.zeros((clss.size, 4 * num_classes), dtype=np.float32)
    bbox_inside_weights = np.zeros(bbox_targets.shape, dtype=np.float32)
    inds = np.where(clss > 0)[0]
    for ind in inds:
        cls = clss[ind]
        start = 4 * cls
        end = start + 4
        bbox_targets[ind, start:end] = bbox_target_data[ind, 1:]
        bbox_inside_weights[ind, start:end] = cfg.TRAIN.BBOX_INSIDE_WEIGHTS
    return bbox_targets, bbox_inside_weights

---