深度神经网络正则化

在深层网络学习中，为了防止过拟合问题，正则化是个有效措施。以下是用tensorflow搭建的小例子，配合tensorboard。

import tensorflow as tf
import os 
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #CPU加速
def  get_weight(shape,lamda,names):
    with tf.name_scope(names+'add'):
        var=tf.Variable(tf.random_normal(shape),dtype=tf.float32,name=names)
    tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(lamda)(var))
    return var

x=tf.placeholder(tf.float32,shape=(None,2),name="input")
y_=tf.placeholder(tf.float32,shape=(None,1),name="label")
batch_size=8

layer_dimension=[2,10,10,10,1]
#神经网络层数
n_layers=len(layer_dimension)
#当前神经网络
cur_layer=x
in_dimension=layer_dimension[0]
#通过一个循环来实现一个5层全连接的神经网络架构
for i in range(1,n_layers):
    out_dimension=layer_dimension[i]    
    with tf.name_scope("layerout"+str(i)):
        weight=get_weight([in_dimension,out_dimension],0.001,"weight"+str(i))
        with tf.name_scope("bias"+str(i)):
            bias=tf.Variable(tf.constant(0.1,shape=[out_dimension]))    
        cur_layer=tf.nn.relu(tf.matmul(cur_layer,weight)+bias,name="relu_out"+str(i))
    in_dimension=out_dimension


with tf.name_scope("output"):
    #标准差
    with tf.name_scope("mse_los"):
        mse_loss=tf.reduce_mean(tf.square(y_-cur_layer))
    tf.add_to_collection('losses',mse_loss)
    #最终目标函数最小化（MSE+每层weight的L2正则化）
    loss=tf.add_n(tf.get_collection('losses'),name="adds_loss")
with tf.Session() as sess:
    #变量初始化
    sess.run(tf.global_variables_initializer())
    ml=sess.run(mse_loss,feed_dict={x:[[2.0,3.9],[1.0,2.2]],y_:[[1],[2]]})
    print(tf.get_collection('losses'))
    print(ml)
    writer=tf.summary.FileWriter("/path/to/log",tf.get_default_graph())

writer.close()