使用TensorFlow训练神经网络识别MNIST数据二

环境：
python 3.7
tensorflow 1.14

代码可在https://github.com/TimeIvyace/MNIST-TensorFlow.git中下载，程序名为train.py。
代码：

# 使用了激活函数去线性化，本身为三层全连接结构，
# 带有指数衰减的学习率以及L2正则化损失函数，
# 同时使用滑动平均模型进行优化。

import tensorflow as tf
old_v = tf.compat.v1.logging.get_verbosity()
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
from tensorflow.examples.tutorials.mnist import input_data


INPUT_NODE = 784  # 输入层的节点数，图片为28*28，为图片的像素
OUTPUT_NODE = 10   # 输出层的节点数，等于类别的数目，需要区分0-9，所以为10类

# 配置神经网络的参数
LAYER1_NODE = 500  # 隐藏层的节点数，此神经网络只有一层隐藏层
BATCH_SIZE = 100  # 一个训练batch中的训练数据个数，数字越小，越接近随机梯度下降，越大越接近梯度下降  ?
LEARNING_RATE_BASE = 0.8  # 基础的学习率
LEARNING_RATE_DECAY = 0.99  # 学习率的衰减率
REGULARIZATION_RATE = 0.0001  # 描述网络复杂度的正则化向在损失函数中的系数
TRAINING_STEPS = 30000  # 训练轮数
MOVING_AVERAGE_DECAY = 0.99  # 滑动平均衰减率

 # 定义了一个使用ReLU的三层全连接神经网络涵涵，返回输出结果
def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):  # 给定神经网络的输入和所有参数
    if avg_class == None:  # 若没有提供滑动平均类，则直接使用参数当前的取值
        layer1 = tf.nn.relu(
            tf.matmul(input_tensor, weights1) + biases1)  # 使用ReLU激活函数,计算隐藏层的前向传播结果
        return tf.matmul(layer1, weights2)+biases2
    else:
        # 首先使用avg_class.average来计算得出变量的滑动平均值
        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
        return tf.matmul(layer1, avg_class.average(weights2))+avg_class.average(biases2)

# 定义训练网络的过程
def train(mnist):
    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')

    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))  # 使用截断正态分布生成隐藏层的参数
    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))  # 较小的正数来初始化偏置项，避免神经元节点输出恒为0的问题

    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))  # 生成输出层的参数
    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))

    # 计算在当前参数下神经网络前向传播的结果，这里的用于计算滑动平均的类为None，所以没有使用滑动平均值
    y = inference(x, None, weights1, biases1, weights2, biases2)

    # 定义存储训练轮数的变量，这个变量不需要被训练
    global_step = tf.Variable(0, trainable=False)

    # 初始化滑动平均类
    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)

    # 在所有代表神经网络参数的变量上使用滑动平均
    variable_averages_op = variable_averages.apply(tf.trainable_varables())

    # 计算使用了滑动平均之后的前向传播结果，滑动平均不会改变变量本身取值，会用一个影子变量来记录
    average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)

    # 计算交叉熵，使用了sparse_softmax_cross_entropy_with_logits，当问题只有一个正确答案时，可以使用这个函数来加速交叉熵的计算。
    # 这个函数的第一个参数是神经网络不包括softmax层的前向传播结果，第二个是训练数据的正确答案,argmax返回最大值的位置
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
    # 计算在当前batch中所有样例的交叉熵平均值
    cross_entropy_mean = tf.reduce_mean(cross_entropy)
    regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)  # 计算L2正则化损失
    regularization = regularizer(weights1) + regularizer(weights2)  # 计算网络的正则化损失
    loss = cross_entropy_mean + regularization  # 总损失为交叉熵损失和正则化损失之和
    learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step,
                                               mnist.train.num_examples/BATCH_SIZE,
                                               LEARNING_RATE_DECAY)  # 设置指数衰减的学习率

    # LEARNING_RATE_BASE为基础学习率，global_step为当前迭代的次数
    # mnist.train.num_examples/BATCH_SIZE为完整的过完所有的训练数据需要的迭代次数
    # LEARNING_RATE_DECAY为学习率衰减速度

    # 使用GradientDescentOptimizer优化算法优化损失函数
    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)

    # 在训练神经网络的时候，每过一遍数据都要通过反向传播来更新参数以及其滑动平均值
    # 为了一次完成多个操作，可以通过tf.control_dependencies和tf.group两种机制来实现
    # train_op = tf.group(train_step, variable_averages_op)  #和下面代码功能一样
    with tf.control_dependencies([train_step, variable_averages_op]):
        train_op = tf.no_op(name = 'train')
    # 计算使用了滑动平均模型的神经网络前向传播结果正确率

    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    # 参数初始化
    with tf.Session() as sess:
        tf.global_variables_initializer().run()

        # 训练神经网络
        for i in range(TRAINING_STEPS):
            if i % 1000 == 0:  # 每1000轮输出一次在验证数据集上的测试结果
                validate_acc = sess.run(accuracy, feed_dict={
                    x: mnist.validation.images, y_:mnist.validation.labels})  # 计算滑动平均模型在验证数据上的结果
                print("After %d training steps, validation accuracy using average model is %g"
                      %(i, validate_acc))

            # 产生训练数据batch,开始训练
            xs, ys = mnist.train.next_batch(BATCH_SIZE)  # 随机抓取训练数据中的100个批处理数据点
            sess.run(train_op, feed_dict={x:xs, y_:ys})

        test_acc = sess.run(accuracy, feed_dict={
            x:mnist.test.images, y_:mnist.test.labels})
        print("After %d training steps, validation accuracy using average model is %g"
              %(TRAINING_STEPS, test_acc))

# 程序主入口
def main(argv=None):
    mnist = input_data.read_data_sets(".", one_hot=True)
    train(mnist)

# TensorFlow提供程序主入口，tf.app.run会调用上面定义的main函数
if __name__ =='__main__':
    tf.app.run()

tf.logging.set_verbosity(old_v)

结果

After 0 training steps, validation accuracy using average model is 0.083
After 1000 training steps, validation accuracy using average model is 0.9774
After 2000 training steps, validation accuracy using average model is 0.9814
After 3000 training steps, validation accuracy using average model is 0.9822
After 4000 training steps, validation accuracy using average model is 0.9824
After 5000 training steps, validation accuracy using average model is 0.9824
After 6000 training steps, validation accuracy using average model is 0.9828
After 7000 training steps, validation accuracy using average model is 0.9828
After 8000 training steps, validation accuracy using average model is 0.9842
After 9000 training steps, validation accuracy using average model is 0.9846
After 10000 training steps, validation accuracy using average model is 0.9834
After 11000 training steps, validation accuracy using average model is 0.9846
After 12000 training steps, validation accuracy using average model is 0.9838