神经网络设计各模块设计

图像标准化

tf.image.per_image_standardization(image)

         这个操作计算(x - mean) / adjusted_stddev，其中mean是图像中所有值的平均值，以及adjusted_stddev = max(stddev, 1.0/sqrt(image.NumElements()))。

          stddev是image中所有值的标准偏差。处理统一图像时，它被限制为零，以防止除以0。

          返回值：与image具有相同形状的标准化的图像。

          参数：image：形状为[height, width, channels]的三维张量

L2正则化

def get_weight(shape):
    return  tf.Variable(tf.random_normal(shape),dtype=tf.float32)
def get_loss(shape,lambda):
    var = get_weight(shape)
    loss = tf.reduce_mean(tf.square(y_ - cur_layer))+tf.contrib.layers.l2_regularizer(lambda)(var))
    return loss

#coding:utf-8

方法二：

import tensorflow as tf

def get_weight(shape,lambda):
    var = tf.Variable(tf.random_normal(shape),dtype=tf.float32)

    tf.add_to_collection("losses",tf.contrib.layers.l2_regularizer(lambda)(var))#把正则化加入集合losses里面

    return var

x = tf.placeholder(tf.float32,shape=(None,2))
y_ = tf.placeholder(tf.float32,shape=(none,1))#真值
weight = get_weight([in_dimension,out_dimension],0.001)
bias = tf.Variable(tf.constant(0.1,shape(out_dimension)))

cur_layer = tf.nn.relu(tf.matmul(x,weight)) + bias)
ses_loss = tf.reduce_mean(tf.square(y_ - cur_layer))#计算最终输出与标准之间的loss

tf.add_to_collenction("losses",ses_loss)#把均方误差也加入到集合里

loss = tf.add_n(tf.get_collection("losses"))    #总的损失
#tf.get_collection返回一个列表,内容是这个集合的所有元素
#add_n()把输入按照元素相加

L1正则化

#L1正则化
tf.contrib.layers.l1_regularizer(0.5)(w)
#L2正则化
tf.contrib.layers.l2_regularizer(0.5)(w)
#将正则化加入到总损失中
loss = tf.reduce_mean(tf.square(y-y_pred) + tf.contrib.layers.l2_regularizer(0.5)(w))

 BatchNorm实现

 

# 实现Batch Normalization
def bn_layer(x, name1, name2, is_training, name='BatchNorm', moving_decay=0.9, eps=1e-5):
    # 获取输入维度并判断是否匹配卷积层(4)或者全连接层(2)
    shape = x.shape
    assert len(shape) in [2, 4]
    param_shape = shape[-1]
    with tf.variable_scope(name):
        # 声明BN中唯一需要学习的两个参数，y=gamma*x+beta
        gamma = tf.get_variable(name1, param_shape, initializer=tf.constant_initializer(1))
        beta = tf.get_variable(name2, param_shape, initializer=tf.constant_initializer(0))
        # 计算当前整个batch的均值与方差
        axes = list(range(len(shape) - 1))  # 特征的通道数
        batch_mean, batch_var = tf.nn.moments(x, axes, name='moments')
        # 采用滑动平均更新均值与方差
        ema = tf.train.ExponentialMovingAverage(moving_decay)
        def mean_var_with_update():
            ema_apply_op = ema.apply([batch_mean, batch_var])
            with tf.control_dependencies([ema_apply_op]):
                return tf.identity(batch_mean), tf.identity(batch_var)

        # 训练时，更新均值与方差，测试时使用之前最后一次保存的均值与方差
        mean, var = tf.cond(tf.equal(is_training, True), mean_var_with_update,
                            lambda: (ema.average(batch_mean), ema.average(batch_var)))

        # 最后执行batch normalization
        return tf.nn.batch_normalization(x, mean, var, beta, gamma, eps)

# 用AdamOptimizer优化器训练
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
    train_step = tf.train.RMSPropOptimizer(learnrate).minimize(zloss)