batch normalization代码实现

BN实现

BN实现的两种

要学会知道BN的推倒公式

import tensorflow as tf
from tensorflow.python.training import moving_averages


class Model:
    def __init__(self,
                 is_train,
                 learning_rate=0.00001,
                 learning_rate_decay_factor=0.9995):
        self.x_ = tf.placeholder(tf.float32, [None, 1, 28, 28])
        self.y_ = tf.placeholder(tf.int32, [None])
        self.keep_prob = tf.placeholder(tf.float32)

        x = tf.reshape(self.x_, [-1, 28, 28, 1])

        # TODO: implement input -- Conv -- BN -- ReLU -- MaxPool -- Conv -- BN -- ReLU -- MaxPool -- Linear -- loss

        channel_size = 8
        kernel_size = 5
        isBN = True

        W_conv1 = weight_variable([kernel_size, kernel_size, 1, channel_size])
        b_conv1 = bias_variable([channel_size])
        h_conv1 = tf.nn.relu(conv2d(x, W_conv1) + b_conv1)

        h_bn1 = batch_normalization_layer(h_conv1, 'bn1', isTrain=is_train,
                                          isBN=isBN)

        relu1 = tf.nn.relu(h_bn1)

        h_pool1 = max_pool(relu1, 2)

        W_conv2 = weight_variable(
            [kernel_size, kernel_size, channel_size, channel_size])
        b_conv2 = bias_variable([channel_size])
        h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)

        h_bn2 = batch_normalization_layer(h_conv2, 'bn2', isTrain=is_train,
                                          isBN=isBN)

        relu2 = tf.nn.relu(h_bn2)

        h_pool2 = max_pool(relu2, 2)

        # reorder so the channels are in the first dimension, x and y follow.
        layer2 = tf.transpose(h_pool2, (0, 3, 1, 2))
        layer2_flat = tf.reshape(layer2, [-1, 7 * 7 * channel_size])

        W_fc1 = weight_variable([7 * 7 * channel_size, 10])
        b_fc1 = bias_variable([10])
        h_fc1 = tf.matmul(layer2_flat, W_fc1) + b_fc1

        #        the 10-class prediction output is named as "logits"
        logits = h_fc1

        self.loss = tf.reduce_mean(
            tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.y_,
                                                           logits=logits))
        self.correct_pred = tf.equal(tf.cast(tf.argmax(logits, 1), tf.int32),
                                     self.y_)
        self.pred = tf.argmax(logits, 1)
        self.acc = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32))

        self.learning_rate = tf.Variable(float(learning_rate), trainable=False,
                                         dtype=tf.float32)
        self.learning_rate_decay_op = self.learning_rate.assign(
            self.learning_rate * learning_rate_decay_factor)

        self.global_step = tf.Variable(0, trainable=False)
        self.params = tf.trainable_variables()
        self.train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(
            self.loss, global_step=self.global_step,
            var_list=self.params)

        self.saver = tf.train.Saver(tf.global_variables(),
                                    write_version=tf.train.SaverDef.V2,
                                    max_to_keep=3, pad_step_number=True,
                                    keep_checkpoint_every_n_hours=1.0)


def weight_variable(shape):  # you can use this func to build new variables
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)


def bias_variable(shape):  # you can use this func to build new variables
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)


def conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')


def max_pool(x, pool_size):
    return tf.nn.max_pool(x, ksize=[1, pool_size, pool_size, 1],
                          strides=[1, pool_size, pool_size, 1], padding='SAME')


def batch_normalization_layer(inputs, name, isTrain=True, isBN=True):
    # TODO: implemented the batch normalization func and applied it on conv and fully-connected layers
    # hint: you can add extra parameters (e.g., shape) if necessary

    '''
    Here I referred this website:
    https://github.com/ry/tensorflow-resnet/blob/master/resnet.py
    '''

    if not isBN:
        return inputs

    with tf.variable_scope(name):
        axis = list(range(len(inputs.get_shape()) - 1))

        mean, variance = tf.nn.moments(inputs, axis)

        beta = tf.get_variable('beta', initializer=tf.zeros_initializer,
                               shape=inputs.get_shape()[-1], dtype=tf.float32)
        gamma = tf.get_variable('gamma', initializer=tf.ones_initializer,
                                shape=inputs.get_shape()[-1], dtype=tf.float32)

        moving_mean = tf.get_variable(name='moving_mean',
                                      initializer=tf.zeros_initializer,
                                      shape=inputs.get_shape()[-1],
                                      dtype=tf.float32, trainable=False)
        moving_variance = tf.get_variable(name='moving_var',
                                          initializer=tf.zeros_initializer,
                                          shape=inputs.get_shape()[-1],
                                          dtype=tf.float32, trainable=False)
        #用于保存
        update_moving_mean = moving_averages.assign_moving_average(moving_mean,
                                                                   mean, 0.999)
        update_moving_variance = moving_averages.assign_moving_average(
            moving_variance, variance, 0.999)

        if isTrain:
            tf.add_to_collection('mean', update_moving_mean)
            tf.add_to_collection('variance', update_moving_variance)
        else:
            mean = update_moving_mean
            variance = update_moving_variance

        inputs = tf.nn.batch_normalization(inputs, mean, variance, beta, gamma,
                                           0.001)

        return inputs

第二种

import tensorflow as tf

def batch_instance_norm(x, scope='batch_instance_norm'):
    with tf.variable_scope(scope):
        ch = x.shape[-1]
        eps = 1e-5

        batch_mean, batch_sigma = tf.nn.moments(x, axes=[0, 1, 2], keep_dims=True)
        x_batch = (x - batch_mean) / (tf.sqrt(batch_sigma + eps))

        ins_mean, ins_sigma = tf.nn.moments(x, axes=[1, 2], keep_dims=True)
        x_ins = (x - ins_mean) / (tf.sqrt(ins_sigma + eps))

        rho = tf.get_variable("rho", [ch], initializer=tf.constant_initializer(1.0), constraint=lambda x: 

tf.clip_by_value(x, clip_value_min=0.0, clip_value_max=1.0))
        gamma = tf.get_variable("gamma", [ch], initializer=tf.constant_initializer(1.0))
        beta = tf.get_variable("beta", [ch], initializer=tf.constant_initializer(0.0))

        x_hat = rho * x_batch + (1 - rho) * x_ins
        x_hat = x_hat * gamma + beta

        return x_hat