ResNet，BN实现

1.ResNet

# bottleneck残差单元模块
def _bottleneck_residual(self, x, in_filter, out_filter, stride,
                         activate_before_residual=False):
    # 是否前置激活(取残差直连之前进行BN和ReLU）
    if activate_before_residual:
        with tf.variable_scope('common_bn_relu'):
            # 先做BN和ReLU激活
            x = self._batch_norm('init_bn', x)
            x = self._relu(x, self.hps.relu_leakiness)
            # 获取残差直连
            orig_x = x
    else:
        with tf.variable_scope('residual_bn_relu'):
            # 获取残差直连
            orig_x = x
            # 后做BN和ReLU激活
            x = self._batch_norm('init_bn', x)
            x = self._relu(x, self.hps.relu_leakiness)

    # 第1子层
    with tf.variable_scope('sub1'):
        # 1x1卷积，使用输入步长，通道数(in_filter -> out_filter/4)
        x = self._conv('conv1', x, 1, in_filter, out_filter / 4, stride)

    # 第2子层
    with tf.variable_scope('sub2'):
        # BN和ReLU激活
        x = self._batch_norm('bn2', x)
        x = self._relu(x, self.hps.relu_leakiness)
        # 3x3卷积，步长为1，通道数不变(out_filter/4)
        x = self._conv('conv2', x, 3, out_filter / 4, out_filter / 4,
                       [1, 1, 1, 1])

    # 第3子层
    with tf.variable_scope('sub3'):
        # BN和ReLU激活
        x = self._batch_norm('bn3', x)
        x = self._relu(x, self.hps.relu_leakiness)
        # 1x1卷积，步长为1，通道数不变(out_filter/4 -> out_filter)
        x = self._conv('conv3', x, 1, out_filter / 4, out_filter,
                       [1, 1, 1, 1])

    # 合并残差层
    with tf.variable_scope('sub_add'):
        # 当通道数有变化时
        if in_filter != out_filter:
            # 1x1卷积，使用输入步长，通道数(in_filter -> out_filter)
            orig_x = self._conv('project', orig_x, 1, in_filter, out_filter,
                                stride)

        # 合并残差
        x += orig_x
    tf.logging.info('image after unit %s', x.get_shape())
    return x

2.BN

# Batch Normalization批归一化
    # ((x-mean)/var)*gamma+beta
def _batch_norm(self, name, x):
    with tf.variable_scope(name):
        # 输入通道维数
        params_shape = [x.get_shape()[-1]]
        # offset
        beta = tf.get_variable('beta',
                               params_shape,
                               tf.float32,
                               initializer=tf.constant_initializer(0.0,
                                                                   tf.float32))
        # scale
        gamma = tf.get_variable('gamma',
                                params_shape,
                                tf.float32,
                                initializer=tf.constant_initializer(1.0,
                                                                    tf.float32))

        if self.mode == 'train':
            # 为每个通道计算均值、标准差
            mean, variance = tf.nn.moments(x, [0, 1, 2], name='moments')
            # 新建或建立测试阶段使用的batch均值、标准差
            moving_mean = tf.get_variable('moving_mean',
                                          params_shape, tf.float32,
                                          initializer=tf.constant_initializer(
                                              0.0, tf.float32),
                                          trainable=False)
            moving_variance = tf.get_variable('moving_variance',
                                              params_shape, tf.float32,
                                              initializer=tf.constant_initializer(
                                                  1.0, tf.float32),
                                              trainable=False)
            # 添加batch均值和标准差的更新操作(滑动平均)
            # moving_mean = moving_mean * decay + mean * (1 - decay)
            # moving_variance = moving_variance * decay + variance * (1 - decay)
            # _extra_train_ops = []使用来存储值的
            self._extra_train_ops.append(
                moving_averages.assign_moving_average(
                    moving_mean, mean, 0.9))
            self._extra_train_ops.append(
                moving_averages.assign_moving_average(
                    moving_variance, variance, 0.9))
        else:
            # 获取训练中积累的batch均值、标准差
            mean = tf.get_variable('moving_mean',
                                   params_shape, tf.float32,
                                   initializer=tf.constant_initializer(0.0,
                                                                       tf.float32),
                                   trainable=False)
            variance = tf.get_variable('moving_variance',
                                       params_shape, tf.float32,
                                       initializer=tf.constant_initializer(
                                           1.0, tf.float32),
                                       trainable=False)
            # 添加到直方图总结
            tf.summary.histogram(mean.op.name, mean)
            tf.summary.histogram(variance.op.name, variance)

        # BN层：((x-mean)/var)*gamma+beta
        y = tf.nn.batch_normalization(x, mean, variance, beta, gamma, 0.001)
        y.set_shape(x.get_shape())
        return y