神经网络 Python TensorFlow MNIST数据集优化（学习笔记）

#完整的TensorFlow程序解决MNIST问题
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

INPUT_NODE=784 #输入层的节点数，图片像素
OUTPUT_NODE=10 #输出层的节点数，类别数目
LAYER1_NODE=500 #隐藏层的节点数

BATCH_SIZE=100 #训练batch中训练数据个数，越小越接近随机梯度下降，越大越接近梯度下降

LEARNING_RATE_BASE=0.8 #基础学习率
LEARNING_RATE_DECAY=0.99 #学习率衰减率
REGULARAZTION_RATE=0.0001 #正则化项在损失函数中的系数
TRAINING_STEPS=5000 #训练轮数
MOVING_AVERAGE_DECAY=0.99 #滑动平均衰减率

#前向传播，隐藏层实现多层网络结构，ReLU激活函数实现去线性化
def inference(input_tensor,avg_class,weights1,biases1,weights2,biases2):
    if avg_class==None:
        layer1=tf.nn.relu(tf.matmul(input_tensor,weights1)+biases1)
        return tf.matmul(layer1,weights2)+biases2 #在计算损失函数时会计算softmax函数，所以不需要加入激活函数
    else:
        layer1=tf.nn.relu(tf.matmul(input_tensor,avg_class.average(weights1))+avg_class.average(biases1))
        return tf.matmul(layer1,avg_class.average(weights2))+avg_class.average(biases2)
    
#训练模型的过程
def train(mnist):
    x=tf.placeholder(tf.float32,[None,INPUT_NODE],name='x-input')
    y_=tf.placeholder(tf.float32,[None,OUTPUT_NODE],name='y-input')
    
    #隐藏层的参数
    weights1=tf.Variable(tf.truncated_normal([INPUT_NODE,LAYER1_NODE],stddev=0.1))
    biases1=tf.Variable(tf.constant(0.1,shape=[LAYER1_NODE]))
    
    #输出层的参数
    weights2=tf.Variable(tf.truncated_normal([LAYER1_NODE,OUTPUT_NODE],stddev=0.1))
    biases2=tf.Variable(tf.constant(0.1,shape=[OUTPUT_NODE]))   
    
    #计算在当前参数下神经网络前向传播的结果
    y=inference(x,None,weights1,biases1,weights2,biases2)
    
    #定义存储训练轮数的变量，定义为不可训练的变量
    global_step=tf.Variable(0,trainable=False)
    
    #初始化滑动平均类
    variable_averages=tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY,global_step)
    
    #在所有代表神经网络参数的变量上使用滑动平均
    variables_averages_op=variable_averages.apply(tf.trainable_variables()) #tf.trainable_variables返回图上集合
    
    #计算使用了滑动平均后的前向传播结果
    average_y=inference(x,variable_averages,weights1,biases1,weights2,biases2)
    
    #损失函数，交叉熵
    cross_entropy=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
    cross_entropy_mean=tf.reduce_mean(cross_entropy)
    
    #L2正则化损失
    regularizer=tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
    regularaztion=regularizer(weights1)+regularizer(weights2)
    
    #总损失
    loss=cross_entropy_mean+regularaztion
    
    #指数衰减的学习率
    learning_rate=tf.train.exponential_decay(LEARNING_RATE_BASE,global_step,mnist.train.num_examples/BATCH_SIZE,LEARNING_RATE_DECAY,staircase=True)
    
    #优化损失函数
    train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
    
    #更新参数，训练模型时，既要通过反向传播来更新神经网络的参数，又要更新每一个参数的滑动平均值
    with tf.control_dependencies([train_step,variables_averages_op]):
        train_op=tf.no_op(name='train')
    
    #计算准确率
    correct_prediction=tf.equal(tf.argmax(average_y,1),tf.argmax(y_,1)) #argmax第二个参数1表示仅在第一维进行，即在每一行选取最大值对应的下标
    accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))

    #开始训练
    with tf.Session() as sess:
        tf.global_variables_initializer().run()
    
        #验证数据
        validata_feed={x:mnist.validation.images,y_:mnist.validation.labels}
    
        #测试数据
        test_feed={x:mnist.test.images,y_:mnist.test.labels}
    
        #迭代训练神经网络
        for i in range(TRAINING_STEPS):
            if i%1000==0:
                validata_acc=sess.run(accuracy,feed_dict=validata_feed)
                print("After %d training step(s),validation accuracy using average model is %g"%(i,validata_acc))
        
            xs,ys=mnist.train.next_batch(BATCH_SIZE)
            sess.run(train_op,feed_dict={x:xs,y_:ys})
    
        test_acc=sess.run(accuracy,feed_dict=test_feed)
        results=sess.run(tf.argmax(average_y,1),feed_dict=test_feed)
        print("After %d training step(s),test accuracy using average model is %g"%(TRAINING_STEPS,test_acc))
        print("Test results:",results)

#主程序
def main(argv=None):
    mnist=input_data.read_data_sets("/MNIST_data/",one_hot=True)
    train(mnist)

#调用上面定义的main函数
if __name__=='__main__':
    main()

After 0 training step(s),validation accuracy using average model is 0.0752
After 1000 training step(s),validation accuracy using average model is 0.9752
After 2000 training step(s),validation accuracy using average model is 0.9802
After 3000 training step(s),validation accuracy using average model is 0.983
After 4000 training step(s),validation accuracy using average model is 0.9832
After 5000 training step(s),test accuracy using average model is 0.9834
Test results: [7 2 1 … 4 5 6]