tensorflow实现正则化 来避免训练过拟合

L1 和 L2 正则化

对神经网络中之后连接权重做限制，比如对只有一个隐层的神经网络做L1正则：

reset_graph()
n_inputs = 28 * 28  # MNIST
n_hidden1 = 300
n_outputs = 10
learning_rate = 0.01

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int64, shape=(None), name="y")

with tf.name_scope("dnn"):
    hidden1 = tf.layers.dense(X, n_hidden1, activation=tf.nn.relu, name="hidden1")
    logits = tf.layers.dense(hidden1, n_outputs, name="outputs")


W1 = tf.get_default_graph().get_tensor_by_name("hidden1/kernel:0")
W2 = tf.get_default_graph().get_tensor_by_name("outputs/kernel:0")

scale = 0.001 # l1正则化参数

with tf.name_scope("loss"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,
                                                             logits=logits)
    base_loss = tf.reduce_mean(xentropy, name="avg_xentropy")
    reg_loss = tf.reduce_sum(tf.abs(W1)) + tf.reduce_sum(tf.abs(W2))
    loss = tf.add(base_loss, scale * reg_loss, name="loss")


with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits,y,1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name="accuracy")
with tf.name_scope("train"):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    training_op = optimizer.minimize(loss)

init = tf.global_variables_initializer()
saver = tf.train.Saver()

训练：

n_epochs = 20
batch_size = 200

with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        for iteration in range(mnist.train.num_examples // batch_size):
            X_batch, y_batch = mnist.train.next_batch(batch_size)
            sess.run(training_op, feed_dict={
  X:X_batch, y:y_batch})
        accuracy_val = accuracy.eval(feed_dict={
  X:mnist.test.images,
                                               y: mnist.test.labels})
        print(epoch, "Test accuracy:", accuracy_val)
    save_path = saver.save(sess,"./my_model_fianl.ckpt")

输出：

0 Test accuracy: 0.8348
1 Test accuracy: 0.8709
2 Test accuracy: 0.8824
3 Test accuracy: 0.8911
4 Test accuracy: 0.8952
5 Test accuracy: 0.8984
6 Test accuracy: 0.9022
7 Test accuracy: 0.9024
8 Test accuracy: 0.9046
9 Test accuracy: 0.906
10 Test accuracy: 0.9063
11 Test accuracy: 0.9074
12 Test accuracy: 0.9083
13 Test accuracy: 0.9073
14 Test accuracy: 0.9066
15 Test accuracy: 0.9077
16 Test accuracy: 0.9065
17 Test accuracy: 0.907
18 Test accuracy: 0.9068
19 Test accuracy: 0.9067

用tf.contrib.layers.l1_regularizer

添加正则项到每个层中，用partial来做：

reset_graph()

n_inputs = 28 * 28  # MNIST
n_hidden1 = 300
n_hidden2 = 50
n_outputs = 10

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int64, shape=(None), name="y")
scale = 0.001
from functools import partial

构建自己的神经网络，用l1做默认的正则化：
收集正则化损失tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)

my_dense_layer = partial(tf.layers.dense,activation=tf.nn.relu,
                        kernel_regularizer=tf.contrib.layers.l1_regularizer(scale))

with tf.name_scope("dnn"):
    hidden1 = my_dense_layer(X, n_hidden1, name="hidden1")
    hidden2 = my_dense_layer(hidden1, n_hidden2,name="hidden2")
    logits = my_dense_layer(hidden2, n_outputs,activation=None,name="outputs")



# 把正则化损失加上
with tf.name_scope("loss"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,logits=logits)
    base_loss = tf.reduce_mean(xentropy,name="avg_xentropy")
    #收集正则化损失
    reg_loss = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
    # tf.add_n: Adds all input tensors element-wise.
    # 参数inputs: A list of `Tensor` objects, each with same shape and type.
    loss = tf.add_n([base_loss] + reg_loss,name="loss")
with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct,tf.float32),name="accuracy")
learning_rate = 0.01
with tf.name_scope("train"):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    training_op = optimizer.minimize(loss)

init = tf