PANET代码 特征融合

################################
#多尺度特征融合
#不同ROI特征融合
###############################
x = KL.Add(name=“mrcnn_mask_add_2_3”)([x2, x3])
x = KL.Add(name=“mrcnn_mask_add_2_4”)([x, x4])
x = KL.Add(name=“mrcnn_mask_add_2_5”)([x, x5])

def fpn_classifier_graph(rois, feature_maps, image_meta,
                         pool_size, num_classes, train_bn=True,
                         fc_layers_size=1024):
    """Builds the computation graph of the feature pyramid network classifier
    and regressor heads.
    rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized
          coordinates.
    feature_maps: List of feature maps from different layers of the pyramid,
                  [P2, P3, P4, P5]. Each has a different resolution.
    image_meta: [batch, (meta data)] Image details. See compose_image_meta()
    pool_size: The width of the square feature map generated from ROI Pooling.
    num_classes: number of classes, which determines the depth of the results
    train_bn: Boolean. Train or freeze Batch Norm layers
    fc_layers_size: Size of the 2 FC layers
    Returns:
        logits: [batch, num_rois, NUM_CLASSES] classifier logits (before softmax)
        probs: [batch, num_rois, NUM_CLASSES] classifier probabilities
        bbox_deltas: [batch, num_rois, NUM_CLASSES, (dy, dx, log(dh), log(dw))] Deltas to apply to
                     proposal boxes
    """
    # ROI Pooling
    # Shape: [batch, num_rois, POOL_SIZE, POOL_SIZE, channels]
    x2 = PyramidROIAlign_AFN([pool_size, pool_size],2,
                        name="roi_align_classifier_2")([rois, image_meta] + feature_maps)

    # Two 1024 FC layers (implemented with Conv2D for consistency)
    x2 = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (pool_size, pool_size), padding="valid"),
                           name="mrcnn_class_conv1_2")(x2)
    x2 = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn1_2')(x2, training=train_bn)
    x2 = KL.Activation('relu')(x2)
    #3
    x3 = PyramidROIAlign_AFN([pool_size, pool_size], 3,
                          name="roi_align_classifier_3")([rois, image_meta] + feature_maps)

    # Two 1024 FC layers (implemented with Conv2D for consistency)
    x3 = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (pool_size, pool_size), padding="valid"),
                            name="mrcnn_class_conv1_3")(x3)
    x3 = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn1_3')(x3, training=train_bn)
    x3 = KL.Activation('relu')(x3)
    #4
    x4 = PyramidROIAlign_AFN([pool_size, pool_size], 4,
                         name="roi_align_classifier_4")([rois, image_meta] + feature_maps)

    # Two 1024 FC layers (implemented with Conv2D for consistency)
    x4 = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (pool_size, pool_size), padding="valid"),
                            name="mrcnn_class_conv1_4")(x4)
    x4 = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn1_4')(x4, training=train_bn)
    x4 = KL.Activation('relu')(x4)
    #5
    x5 = PyramidROIAlign_AFN([pool_size, pool_size], 5,
                          name="roi_align_classifier_5")([rois, image_meta] + feature_maps)

    # Two 1024 FC layers (implemented with Conv2D for consistency)
    x5 = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (pool_size, pool_size), padding="valid"),
                            name="mrcnn_class_conv1_5")(x5)
    x5 = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn1_5')(x5, training=train_bn)
    x5 = KL.Activation('relu')(x5)
    ################################
    #多尺度特征融合
    #不同ROI特征融合
    ###############################
    x = KL.Add(name="mrcnn_mask_add_2_3")([x2, x3])
    x = KL.Add(name="mrcnn_mask_add_2_4")([x, x4])
    x = KL.Add(name="mrcnn_mask_add_2_5")([x, x5])

    x = KL.TimeDistributed(KL.Conv2D(fc_layers_size, (1, 1)),
                           name="mrcnn_class_conv2")(x)
    x = KL.TimeDistributed(BatchNorm(), name='mrcnn_class_bn2')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    shared = KL.Lambda(lambda x: K.squeeze(K.squeeze(x, 3), 2),
                       name="pool_squeeze")(x)

    # Classifier head
    mrcnn_class_logits = KL.TimeDistributed(KL.Dense(num_classes),
                                            name='mrcnn_class_logits')(shared)
    mrcnn_probs = KL.TimeDistributed(KL.Activation("softmax"),
                                     name="mrcnn_class")(mrcnn_class_logits)

    # BBox head
    # [batch, num_rois, NUM_CLASSES * (dy, dx, log(dh), log(dw))]
    x = KL.TimeDistributed(KL.Dense(num_classes * 4, activation='linear'),
                           name='mrcnn_bbox_fc')(shared)
    # Reshape to [batch, num_rois, NUM_CLASSES, (dy, dx, log(dh), log(dw))]
    s = K.int_shape(x)
    mrcnn_bbox = KL.Reshape((s[1], num_classes, 4), name="mrcnn_bbox")(x)

    return mrcnn_class_logits, mrcnn_probs, mrcnn_bbox

  for i, level in enumerate(range(2, 6)):
            ix = tf.where(tf.equal(roi_level, level))
            level_boxes = tf.gather_nd(boxes, ix)

            # Box indices for crop_and_resize.
            box_indices = tf.cast(ix[:, 0], tf.int32)

            # Keep track of which box is mapped to which level
            box_to_level.append(ix)

            # Stop gradient propogation to ROI proposals
            level_boxes = tf.stop_gradient(level_boxes)
            box_indices = tf.stop_gradient(box_indices)

            # Crop and Resize
            # From Mask R-CNN paper: "We sample four regular locations, so
            # that we can evaluate either max or average pooling. In fact,
            # interpolating only a single value at each bin center (without
            # pooling) is nearly as effective."
            #
            # Here we use the simplified approach of a single value per bin,
            # which is how it's done in tf.crop_and_resize()
            # Result: [batch * num_boxes, pool_height, pool_width, channels]
            pooled.append(tf.image.crop_and_resize(
                feature_maps[i], level_boxes, box_indices, self.pool_shape,
                method="bilinear"))

        # Pack pooled features into one tensor
        pooled = tf.concat(pooled, axis=0)

def build_fpn_mask_graph(rois, feature_maps, image_meta,
                         pool_size, num_classes, train_bn=True):
    """Builds the computation graph of the mask head of Feature Pyramid Network.
    rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized
          coordinates.
    feature_maps: List of feature maps from different layers of the pyramid,
                  [P2, P3, P4, P5]. Each has a different resolution.
    image_meta: [batch, (meta data)] Image details. See compose_image_meta()
    pool_size: The width of the square feature map generated from ROI Pooling.
    num_classes: number of classes, which determines the depth of the results
    train_bn: Boolean. Train or freeze Batch Norm layers
    Returns: Masks [batch, num_rois, MASK_POOL_SIZE, MASK_POOL_SIZE, NUM_CLASSES]
    """
    # ROI Pooling
    # Shape: [batch, num_rois, MASK_POOL_SIZE, MASK_POOL_SIZE, channels]
    x = PyramidROIAlign([pool_size, pool_size],
                        name="roi_align_mask")([rois, image_meta] + feature_maps)

    # Conv layers
    x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"),
                           name="mrcnn_mask_conv1")(x)
    x = KL.TimeDistributed(BatchNorm(),
                           name='mrcnn_mask_bn1')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"),
                           name="mrcnn_mask_conv2")(x)
    x = KL.TimeDistributed(BatchNorm(),
                           name='mrcnn_mask_bn2')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"),
                           name="mrcnn_mask_conv3")(x)
    x = KL.TimeDistributed(BatchNorm(),
                           name='mrcnn_mask_bn3')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    x1 = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"),
                           name="mrcnn_mask_conv4_fc")(x)
    x1 = KL.TimeDistributed(BatchNorm(),
                           name='mrcnn_mask_conv4bn')(x1, training=train_bn)
    x1 = KL.Activation('relu')(x1)

    x1 = KL.TimeDistributed(KL.Conv2D(256, (3, 3),strides=(2,2), padding="same"),
                           name="mrcnn_mask_conv5_fc")(x1)
    x1 = KL.TimeDistributed(BatchNorm(),
                            name='mrcnn_mask_conv5bn')(x1, training=train_bn)
    x1 = KL.Activation('relu')(x1)

    #x1 = KL.TimeDistributed(KL.Dense(256*4*4,activation="sigmoid"),
    #                       name="mrcnn_mask_fc")(x1)
    x1 = KL.TimeDistributed(KL.Flatten())(x1)
    x1 = KL.TimeDistributed(KL.Dense(28*28*num_classes),name='mrcnn_mask_fc_logits')(x1)

    x1 = KL.Activation("softmax",name="mrcnn_class_fc")(x1)



    s = K.int_shape(x1)
    x1 = KL.Reshape(( s[1],28,28, num_classes), name="mrcnn_mask_fc_reshape")(x1)
    #x1 = KL.TimeDistributed(KL.Reshape((14,14)),name="mrcnn_mask_fc_reshape")(x1)

    x = KL.TimeDistributed(KL.Conv2D(256, (3, 3), padding="same"),
                           name="mrcnn_mask_conv4")(x)
    x = KL.TimeDistributed(BatchNorm(),
                           name='mrcnn_mask_bn4')(x, training=train_bn)
    x = KL.Activation('relu')(x)

    x = KL.TimeDistributed(KL.Conv2DTranspose(256, (2, 2), strides=2, activation="relu"),
                           name="mrcnn_mask_deconv")(x)


    x = KL.TimeDistributed(KL.Conv2D(num_classes, (1, 1), strides=1, activation="softmax"),
                           name="mrcnn_mask")(x)
    x = KL.Add(name="mrcnn_mask_add")([x, x1])
    x = KL.Activation('tanh',name="mrcnn_masksoftmax")(x)


    return x