神经网络 Python TensorFlow MNIST数据集最佳实践（学习笔记）

加入了变量管理机制和持久化机制
前向传播 单独库函数
训练和测试 分为两个独立的程序

#前向传播
import tensorflow as tf

INPUT_NODE=784 #输入层的节点数，图片像素
OUTPUT_NODE=10 #输出层的节点数，类别数目
LAYER1_NODE=500 #隐藏层的节点数

#通过tf.get_variable获取变量。在训练时创建，在测试时加载
def get_weight_variable(shape,regularizer):
    weights=tf.get_variable("weights",shape,initializer=tf.truncated_normal_initializer(stddev=0.1))
    #当给出正则化生成函数时，将正则化损失加入losses集合
    if regularizer!=None:
        tf.add_to_collection('losses',regularizer(weights))
    return weights

#定义神经网络的前向传播过程
def inference(input_tensor,regularizer):
    #第一层
    with tf.variable_scope('layer1'):
        weights=get_weight_variable([INPUT_NODE,LAYER1_NODE],regularizer)
        biases=tf.get_variable("biases",[LAYER1_NODE],initializer=tf.constant_initializer(0.0))
        layer1=tf.nn.relu(tf.matmul(input_tensor,weights)+biases)
    #第二层  
    with tf.variable_scope('layer2'):
        weights=get_weight_variable([LAYER1_NODE,OUTPUT_NODE],regularizer)
        biases=tf.get_variable("biases",[OUTPUT_NODE],initializer=tf.constant_initializer(0.0))
        layer2=tf.matmul(layer1,weights)+biases
        
    return layer2

#训练程序
import os
from tensorflow.examples.tutorials.mnist import input_data

BATCH_SIZE=100 #训练batch中训练数据个数，越小越接近随机梯度下降，越大越接近梯度下降

LEARNING_RATE_BASE=0.8 #基础学习率
LEARNING_RATE_DECAY=0.99 #学习率衰减率
REGULARAZTION_RATE=0.0001 #正则化项在损失函数中的系数
TRAINING_STEPS=5000 #训练轮数
MOVING_AVERAGE_DECAY=0.99 #滑动平均衰减率

#模型保存的路径和文件名
MODEL_SAVE_PATH="E:/.../"
MODEL_NAME="model5.ckpt"

def train(mnist):
    x=tf.placeholder(tf.float32,[None,INPUT_NODE],name='x-input')
    y_=tf.placeholder(tf.float32,[None,OUTPUT_NODE],name='y-input')
    
    #正则化
    regularizer=tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
    #前向传播过程
    y=inference(x,regularizer)
    
    #定义存储训练轮数的变量，定义为不可训练的变量
    global_step=tf.Variable(0,trainable=False)
    
    #滑动平均
    variable_averages=tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY,global_step)
    variables_averages_op=variable_averages.apply(tf.trainable_variables())
    
    #损失函数
    cross_entropy=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
    cross_entropy_mean=tf.reduce_mean(cross_entropy)
    loss=cross_entropy_mean+tf.add_n(tf.get_collection('losses'))
    
    #学习率
    learning_rate=tf.train.exponential_decay(LEARNING_RATE_BASE,global_step,mnist.train.num_examples/BATCH_SIZE,LEARNING_RATE_DECAY,staircase=True)
    
    #优化损失函数
    train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
    
    #更新参数，训练模型时，既要通过反向传播来更新神经网络的参数，又要更新每一个参数的滑动平均值
    with tf.control_dependencies([train_step,variables_averages_op]):
        train_op=tf.no_op(name='train')
        
    #初始化TensorFlow持久化类
    saver=tf.train.Saver()
    with tf.Session() as sess:
        tf.global_variables_initializer().run()
        
        #在训练中不再测试模型在验证集上的表现
        for i in range(TRAINING_STEPS):
            xs,ys=mnist.train.next_batch(BATCH_SIZE)
            _,loss_value,step=sess.run([train_op,loss,global_step],feed_dict={x:xs,y_:ys})
            if i%1000==0:
                #输出当前的训练情况
                print("After %d training step(s),loss on training batch is %g"%(step,loss_value))
                #保存当前的模型。给出global_step参数，让每个文件名末尾加上训练的轮数
                saver.save(sess,os.path.join(MODEL_SAVE_PATH,MODEL_NAME),global_step=global_step)

#主程序
def main(argv=None):
    mnist=input_data.read_data_sets("/MNIST_data/",one_hot=True)
    train(mnist)

#调用上面定义的main函数
if __name__=='__main__':
    main()

After 1 training step(s),loss on training batch is 2.61085
After 1001 training step(s),loss on training batch is 0.272755
After 2001 training step(s),loss on training batch is 0.230244
After 3001 training step(s),loss on training batch is 0.143116
After 4001 training step(s),loss on training batch is 0.117189

#测试程序
import time

#每10秒加载一次最新的模型，并在测试数据上测试最新模型的正确率
EVAL_INTERVAL_SECS=10

def evaluate(mnist):
    with tf.Graph().as_default() as g:
        x=tf.placeholder(tf.float32,[None,INPUT_NODE],name='x-input')
        y_=tf.placeholder(tf.float32,[None,OUTPUT_NODE],name='y-input')
        
        #验证数据
        validata_feed={x:mnist.validation.images,y_:mnist.validation.labels}
        
        #计算前向传播的结果，测试时不关注正则化损失的值
        y=inference(x,None)
        
        #使用前向传播的结果计算正确率
        correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(y_,1)) #argmax第二个参数1表示仅在第一维进行，即在每一行选取最大值对应的下标
        accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
        
        #通过变量重命名加载模型。共用前向传播过程
        variable_averages=tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY)
        variables_to_restore=variable_averages.variables_to_restore()
        saver=tf.train.Saver(variables_to_restore)
        
        #每隔EVAL_INTERVAL_SECS秒调用一次计算正确率的过程以检测训练过程中正确率的变化
        while True:
            with tf.Session() as sess:
                #找到目录中最新模型的文件名
                ckpt=tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
                if ckpt and ckpt.model_checkpoint_path:
                    #加载模型
                    saver.restore(sess,ckpt.model_checkpoint_path)
                    #通过文件名得到模型保存时迭代的轮数
                    global_step=ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
                    accuracy_score=sess.run(accuracy,feed_dict=validata_feed)
                    print("After %s training step(s),validation accuracy = %g"%(global_step,accuracy_score))
                else:
                    print("No checkpoint file found")
                    return 
            time.sleep(EVAL_INTERVAL_SECS)
            
#主程序
def main(argv=None):
    mnist=input_data.read_data_sets("/MNIST_data/",one_hot=True)
    evaluate(mnist)

#调用上面定义的main函数
if __name__=='__main__':
    main()                               

INFO:tensorflow:Restoring parameters from E:/Python/TensorFlow实战Google深度学习框架/练习/chap05/model5.ckpt-4001
After 4001 training step(s),validation accuracy = 0.9828
INFO:tensorflow:Restoring parameters from E:/Python/TensorFlow实战Google深度学习框架/练习/chap05/model5.ckpt-4001
After 4001 training step(s),validation accuracy = 0.9828
INFO:tensorflow:Restoring parameters from E:/Python/TensorFlow实战Google深度学习框架/练习/chap05/model5.ckpt-4001
After 4001 training step(s),validation accuracy = 0.9828
……