## 01 mnist数据读取

# 《TensorFlow实战Google深度学习框架》05 minist数字识别问题
# win10 Tensorflow1.0.1 python3.5.3
# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1
# filename:ts05.01.py # mnist数据读取

import tensorflow as tf

# 在Yann LeCun教授的网站中（http://yann.lecun.com/exdb/mnist ） 对MNIST数据集做出了详细的介绍。
# 1. 读取数据集，第一次TensorFlow会自动下载数据集到下面的路径中
from tensorflow.examples.tutorials.mnist import input_data

# 2. 数据集会自动被分成3个子集，train、validation和test。以下代码会显示数据集的大小。
print("Training data size: ", mnist.train.num_examples)
print("Validating data size: ", mnist.validation.num_examples)
print("Testing data size: ", mnist.test.num_examples)

# 3. 查看training数据集中某个成员的像素矩阵生成的一维数组和其属于的数字标签。
print("Example training data: ", mnist.train.images[0])
print("Example training data label: ", mnist.train.labels[0])

# 4. 使用mnist.train.next_batch来实现随机梯度下降。
batch_size = 100
xs, ys = mnist.train.next_batch(batch_size)    # 从train的集合中选取batch_size个训练数据。
print("X shape:", xs.shape)
print("Y shape:", ys.shape)

'''
Extracting ../../datasets/MNIST_data/train-images-idx3-ubyte.gz
Extracting ../../datasets/MNIST_data/train-labels-idx1-ubyte.gz
Extracting ../../datasets/MNIST_data/t10k-images-idx3-ubyte.gz
Extracting ../../datasets/MNIST_data/t10k-labels-idx1-ubyte.gz
Training data size:  55000
Validating data size:  5000
Testing data size:  10000
Example training data:  [ 0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.38039219  0.37647063
0.3019608   0.46274513  0.2392157   0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.35294119  0.5411765
0.92156869  0.92156869  0.92156869  0.92156869  0.92156869  0.92156869
0.98431379  0.98431379  0.97254908  0.99607849  0.96078438  0.92156869
0.74509805  0.08235294  0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.
0.54901963  0.98431379  0.99607849  0.99607849  0.99607849  0.99607849
0.99607849  0.99607849  0.99607849  0.99607849  0.99607849  0.99607849
0.99607849  0.99607849  0.99607849  0.99607849  0.74117649  0.09019608
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.88627458  0.99607849  0.81568635
0.78039223  0.78039223  0.78039223  0.78039223  0.54509807  0.2392157
0.2392157   0.2392157   0.2392157   0.2392157   0.50196081  0.8705883
0.99607849  0.99607849  0.74117649  0.08235294  0.          0.          0.
0.          0.          0.          0.          0.          0.
0.14901961  0.32156864  0.0509804   0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.13333334  0.83529419  0.99607849  0.99607849  0.45098042  0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.32941177  0.99607849  0.99607849  0.91764712  0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.32941177  0.99607849  0.99607849  0.91764712  0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.41568631  0.6156863   0.99607849  0.99607849  0.95294124  0.20000002
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.09803922  0.45882356  0.89411771
0.89411771  0.89411771  0.99215692  0.99607849  0.99607849  0.99607849
0.99607849  0.94117653  0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.26666668  0.4666667   0.86274517
0.99607849  0.99607849  0.99607849  0.99607849  0.99607849  0.99607849
0.99607849  0.99607849  0.99607849  0.55686277  0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.14509805  0.73333335  0.99215692
0.99607849  0.99607849  0.99607849  0.87450987  0.80784321  0.80784321
0.29411766  0.26666668  0.84313732  0.99607849  0.99607849  0.45882356
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.44313729
0.8588236   0.99607849  0.94901967  0.89019614  0.45098042  0.34901962
0.12156864  0.          0.          0.          0.          0.7843138
0.99607849  0.9450981   0.16078432  0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.66274512  0.99607849  0.6901961   0.24313727  0.          0.
0.          0.          0.          0.          0.          0.18823531
0.90588242  0.99607849  0.91764712  0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.07058824  0.48627454  0.          0.          0.
0.          0.          0.          0.          0.          0.
0.32941177  0.99607849  0.99607849  0.65098041  0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.54509807  0.99607849  0.9333334   0.22352943  0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.
0.82352948  0.98039222  0.99607849  0.65882355  0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.94901967  0.99607849  0.93725497  0.22352943  0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.
0.34901962  0.98431379  0.9450981   0.33725491  0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.
0.01960784  0.80784321  0.96470594  0.6156863   0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.01568628  0.45882356  0.27058825  0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.
0.          0.          0.          0.          0.          0.          0.        ]
Example training data label:  [ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]
X shape: (100, 784)
Y shape: (100, 10)
'''

## 02 全模型

# 《TensorFlow实战Google深度学习框架》05 minist数字识别问题
# win10 Tensorflow1.0.1 python3.5.3
# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1
# filename:ts05.02.py # TensorFlow训练神经网络--全模型

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

# 1.设置输入和输出节点的个数,配置神经网络的参数
INPUT_NODE = 784  # 输入节点
OUTPUT_NODE = 10  # 输出节点
LAYER1_NODE = 500  # 隐藏层数

BATCH_SIZE = 100  # 每次batch打包的样本个数

# 模型相关的参数
LEARNING_RATE_BASE = 0.8
LEARNING_RATE_DECAY = 0.99
REGULARAZTION_RATE = 0.0001
TRAINING_STEPS = 5000
MOVING_AVERAGE_DECAY = 0.99

# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数。
def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):
# 不使用滑动平均类
if avg_class == None:
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
return tf.matmul(layer1, weights2) + biases2
else:
# 使用滑动平均类
layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)

# 3. 定义训练过程
def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')
# 生成隐藏层的参数。
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))
# 生成输出层的参数。
weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))

# 计算不含滑动平均类的前向传播结果
y = inference(x, None, weights1, biases1, weights2, biases2)

# 定义训练轮数及相关的滑动平均类
global_step = tf.Variable(0, trainable=False)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)

# 计算交叉熵及其平均值
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)

# 损失函数的计算
regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
regularaztion = regularizer(weights1) + regularizer(weights2)
loss = cross_entropy_mean + regularaztion

# 设置指数衰减的学习率。
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)

# 优化损失函数

# 反向传播更新参数和更新每一个参数的滑动平均值
with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')

# 计算正确率
correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# 初始化回话并开始训练过程。
with tf.Session() as sess:
tf.global_variables_initializer().run()
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
test_feed = {x: mnist.test.images, y_: mnist.test.labels}

# 循环的训练神经网络。
for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))

xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x: xs, y_: ys})

test_acc = sess.run(accuracy, feed_dict=test_feed)
print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))

# 4. 主程序入口，这里设定模型训练次数为5000次。
def main(argv=None):
train(mnist)

if __name__=='__main__':
main()

'''
Extracting ../../../datasets/MNIST_data\train-images-idx3-ubyte.gz
Extracting ../../../datasets/MNIST_data\train-labels-idx1-ubyte.gz
Extracting ../../../datasets/MNIST_data\t10k-images-idx3-ubyte.gz
Extracting ../../../datasets/MNIST_data\t10k-labels-idx1-ubyte.gz
After 0 training step(s), validation accuracy using average model is 0.1284
After 1000 training step(s), validation accuracy using average model is 0.9764
After 2000 training step(s), validation accuracy using average model is 0.9806
After 3000 training step(s), validation accuracy using average model is 0.9818
After 4000 training step(s), validation accuracy using average model is 0.9822
After 5000 training step(s), test accuracy using average model is 0.9822
'''

## 03 不使用正则化

# 《TensorFlow实战Google深度学习框架》05 minist数字识别问题
# win10 Tensorflow1.0.1 python3.5.3
# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1
# filename:ts05.03.py # TensorFlow训练神经网络--不使用正则化

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

# 1.设置输入和输出节点的个数,配置神经网络的参数。
INPUT_NODE = 784  # 输入节点
OUTPUT_NODE = 10  # 输出节点
LAYER1_NODE = 500  # 隐藏层数

BATCH_SIZE = 100  # 每次batch打包的样本个数

# 模型相关的参数
LEARNING_RATE_BASE = 0.8
LEARNING_RATE_DECAY = 0.99

TRAINING_STEPS = 5000
MOVING_AVERAGE_DECAY = 0.99

# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数
def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):
# 不使用滑动平均类
if avg_class == None:
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
return tf.matmul(layer1, weights2) + biases2
else:
# 使用滑动平均类
layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)

# 3. 定义训练过程。
def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')
# 生成隐藏层的参数。
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))
# 生成输出层的参数。
weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))

# 计算不含滑动平均类的前向传播结果
y = inference(x, None, weights1, biases1, weights2, biases2)

# 定义训练轮数及相关的滑动平均类
global_step = tf.Variable(0, trainable=False)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)

# 计算交叉熵及其平均值
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)

# 损失函数的计算
loss = cross_entropy_mean

# 设置指数衰减的学习率。
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)

# 优化损失函数

# 反向传播更新参数和更新每一个参数的滑动平均值
with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')

# 计算正确率
correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# 初始化回话并开始训练过程。
with tf.Session() as sess:
tf.global_variables_initializer().run()
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
test_feed = {x: mnist.test.images, y_: mnist.test.labels}

# 循环的训练神经网络。
for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))

xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x: xs, y_: ys})

test_acc = sess.run(accuracy, feed_dict=test_feed)
print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))

# 4. 主程序入口，这里设定模型训练次数为5000次。
def main(argv=None):
train(mnist)

if __name__=='__main__':
main()

'''
Extracting ../../../datasets/MNIST_data\train-images-idx3-ubyte.gz
Extracting ../../../datasets/MNIST_data\train-labels-idx1-ubyte.gz
Extracting ../../../datasets/MNIST_data\t10k-images-idx3-ubyte.gz
Extracting ../../../datasets/MNIST_data\t10k-labels-idx1-ubyte.gz
After 0 training step(s), validation accuracy using average model is 0.1138
After 1000 training step(s), validation accuracy using average model is 0.9754
After 2000 training step(s), validation accuracy using average model is 0.979
After 3000 training step(s), validation accuracy using average model is 0.982
After 4000 training step(s), validation accuracy using average model is 0.9826
After 5000 training step(s), test accuracy using average model is 0.9817
'''

## 04 不使用指数衰减的学习率

# 《TensorFlow实战Google深度学习框架》05 minist数字识别问题
# win10 Tensorflow1.0.1 python3.5.3
# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1
# filename:ts05.04.py # TensorFlow训练神经网络--不使用指数衰减的学习率

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

# 1.设置输入和输出节点的个数,配置神经网络的参数
INPUT_NODE = 784  # 输入节点
OUTPUT_NODE = 10  # 输出节点
LAYER1_NODE = 500  # 隐藏层数

BATCH_SIZE = 100  # 每次batch打包的样本个数

# 模型相关的参数
LEARNING_RATE = 0.1
REGULARAZTION_RATE = 0.0001
TRAINING_STEPS = 5000
MOVING_AVERAGE_DECAY = 0.99

# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数
def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):
# 不使用滑动平均类
if avg_class == None:
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
return tf.matmul(layer1, weights2) + biases2
else:
# 使用滑动平均类
layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)
# 3. 定义训练过程。
def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')
# 生成隐藏层的参数。
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))
# 生成输出层的参数。
weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))

# 计算不含滑动平均类的前向传播结果
y = inference(x, None, weights1, biases1, weights2, biases2)

# 定义训练轮数及相关的滑动平均类
global_step = tf.Variable(0, trainable=False)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)

# 计算交叉熵及其平均值
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)

# 损失函数的计算
regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
regularaztion = regularizer(weights1) + regularizer(weights2)
loss = cross_entropy_mean + regularaztion

# 优化损失函数

# 反向传播更新参数和更新每一个参数的滑动平均值
with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')

# 计算正确率
correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# 初始化回话并开始训练过程。
with tf.Session() as sess:
tf.global_variables_initializer().run()
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
test_feed = {x: mnist.test.images, y_: mnist.test.labels}

# 循环的训练神经网络。
for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))

xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x: xs, y_: ys})

test_acc = sess.run(accuracy, feed_dict=test_feed)
print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))

# 4. 主程序入口，这里设定模型训练次数为5000次。
def main(argv=None):
train(mnist)

if __name__=='__main__':
main()
'''
Extracting ../../../datasets/MNIST_data\train-images-idx3-ubyte.gz
Extracting ../../../datasets/MNIST_data\train-labels-idx1-ubyte.gz
Extracting ../../../datasets/MNIST_data\t10k-images-idx3-ubyte.gz
Extracting ../../../datasets/MNIST_data\t10k-labels-idx1-ubyte.gz
After 0 training step(s), validation accuracy using average model is 0.1076
After 1000 training step(s), validation accuracy using average model is 0.9462
After 2000 training step(s), validation accuracy using average model is 0.9636
After 3000 training step(s), validation accuracy using average model is 0.969
After 4000 training step(s), validation accuracy using average model is 0.9716
After 5000 training step(s), test accuracy using average model is 0.973
'''

## 05 不使用激活函数

# 《TensorFlow实战Google深度学习框架》05 minist数字识别问题
# win10 Tensorflow1.0.1 python3.5.3
# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1
# filename:ts05.05.py # TensorFlow训练神经网络--不使用激活函数

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

# 1.设置输入和输出节点的个数,配置神经网络的参数
INPUT_NODE = 784  # 输入节点
OUTPUT_NODE = 10  # 输出节点
LAYER1_NODE = 500  # 隐藏层数

BATCH_SIZE = 100  # 每次batch打包的样本个数

# 模型相关的参数
LEARNING_RATE_BASE = 0.8
LEARNING_RATE_DECAY = 0.99
REGULARAZTION_RATE = 0.0001
TRAINING_STEPS = 5000
MOVING_AVERAGE_DECAY = 0.99

# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数
def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):
# 不使用滑动平均类
if avg_class == None:
layer1 = tf.matmul(input_tensor, weights1) + biases1
return tf.matmul(layer1, weights2) + biases2
else:
# 使用滑动平均类
layer1 = tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1)
return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)

# 3. 定义训练过程
def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')
# 生成隐藏层的参数。
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))
# 生成输出层的参数。
weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))

# 计算不含滑动平均类的前向传播结果
y = inference(x, None, weights1, biases1, weights2, biases2)

# 定义训练轮数及相关的滑动平均类
global_step = tf.Variable(0, trainable=False)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)

# 计算交叉熵及其平均值
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)

# 损失函数的计算
regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
regularaztion = regularizer(weights1) + regularizer(weights2)
loss = cross_entropy_mean + regularaztion

# 设置指数衰减的学习率。
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)

# 优化损失函数

# 反向传播更新参数和更新每一个参数的滑动平均值
with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')

# 计算正确率
correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# 初始化回话并开始训练过程。
with tf.Session() as sess:
tf.global_variables_initializer().run()
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
test_feed = {x: mnist.test.images, y_: mnist.test.labels}

# 循环的训练神经网络。
for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))

xs,ys=mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op,feed_dict={x:xs,y_:ys})

test_acc=sess.run(accuracy,feed_dict=test_feed)
print(("After %d training step(s), test accuracy using average model is %g" %(TRAINING_STEPS, test_acc)))

# 4. 主程序入口，这里设定模型训练次数为5000次
def main(argv=None):
train(mnist)

if __name__=='__main__':
main()

'''
After 0 training step(s), validation accuracy using average model is 0.08
After 1000 training step(s), validation accuracy using average model is 0.0958
After 2000 training step(s), validation accuracy using average model is 0.0958
After 3000 training step(s), validation accuracy using average model is 0.0958
After 4000 training step(s), validation accuracy using average model is 0.0958
After 5000 training step(s), test accuracy using average model is 0.098
'''

## 06 不使用隐藏层

# 《TensorFlow实战Google深度学习框架》05 minist数字识别问题
# win10 Tensorflow1.0.1 python3.5.3
# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1
# filename:ts05.06.py # TensorFlow训练神经网络--不使用隐藏层

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

# 1.设置输入和输出节点的个数,配置神经网络的参数
INPUT_NODE = 784     # 输入节点
OUTPUT_NODE = 10     # 输出节点

BATCH_SIZE = 100     # 每次batch打包的样本个数

# 模型相关的参数
LEARNING_RATE_BASE = 0.8
LEARNING_RATE_DECAY = 0.99
REGULARAZTION_RATE = 0.0001
TRAINING_STEPS = 5000
MOVING_AVERAGE_DECAY = 0.99

# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数
def inference(input_tensor, avg_class, weights1, biases1):
# 不使用滑动平均类
if avg_class == None:
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
return layer1
else:
# 使用滑动平均类
layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
return layer1

# 3. 定义训练过程
def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')

# 生成输出层的参数。
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, OUTPUT_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))

# 计算不含滑动平均类的前向传播结果
y = inference(x, None, weights1, biases1)

# 定义训练轮数及相关的滑动平均类
global_step = tf.Variable(0, trainable=False)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
average_y = inference(x, variable_averages, weights1, biases1)

# 计算交叉熵及其平均值
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)

# 损失函数的计算
regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
regularaztion = regularizer(weights1)
loss = cross_entropy_mean + regularaztion

# 设置指数衰减的学习率。
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)

# 优化损失函数

# 反向传播更新参数和更新每一个参数的滑动平均值
with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')

# 计算正确率
correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# 初始化回话并开始训练过程。
with tf.Session() as sess:
tf.global_variables_initializer().run()
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
test_feed = {x: mnist.test.images, y_: mnist.test.labels}

# 循环的训练神经网络。
for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))

xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x: xs, y_: ys})

test_acc = sess.run(accuracy, feed_dict=test_feed)
print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))

# 4. 主程序入口，这里设定模型训练次数为5000次
def main(argv=None):
train(mnist)

if __name__=='__main__':
main()

'''
After 0 training step(s), validation accuracy using average model is 0.1166
After 1000 training step(s), validation accuracy using average model is 0.6498
After 2000 training step(s), validation accuracy using average model is 0.7536
After 3000 training step(s), validation accuracy using average model is 0.7546
After 4000 training step(s), validation accuracy using average model is 0.7552
After 5000 training step(s), test accuracy using average model is 0.7501
'''

## 07 不使用滑动平均

# 《TensorFlow实战Google深度学习框架》05 minist数字识别问题
# win10 Tensorflow1.0.1 python3.5.3
# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1
# filename:ts05.07.py # TensorFlow训练神经网络--不使用滑动平均

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

# 1.设置输入和输出节点的个数,配置神经网络的参数
INPUT_NODE = 784  # 输入节点
OUTPUT_NODE = 10  # 输出节点
LAYER1_NODE = 500  # 隐藏层数

BATCH_SIZE = 100  # 每次batch打包的样本个数

# 模型相关的参数
LEARNING_RATE_BASE = 0.8
LEARNING_RATE_DECAY = 0.99
REGULARAZTION_RATE = 0.0001
TRAINING_STEPS = 5000

# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数
def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):
# 不使用滑动平均类
if avg_class == None:
layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
return tf.matmul(layer1, weights2) + biases2
else:
# 使用滑动平均类
layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))
return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)

# 3. 定义训练过程
def train(mnist):
x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')
y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')
# 生成隐藏层的参数。
weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))
biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))
# 生成输出层的参数。
weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))
biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))

# 计算不含滑动平均类的前向传播结果
y = inference(x, None, weights1, biases1, weights2, biases2)

# 定义训练轮数及相关的滑动平均类
global_step = tf.Variable(0, trainable=False)

# 计算交叉熵及其平均值
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)

# 损失函数的计算
regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
regularaztion = regularizer(weights1) + regularizer(weights2)
loss = cross_entropy_mean + regularaztion

# 设置指数衰减的学习率。
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)

# 优化损失函数

# 反向传播更新参数
with tf.control_dependencies([train_step]):
train_op = tf.no_op(name='train')

# 计算正确率
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# 初始化回话并开始训练过程。
with tf.Session() as sess:
tf.global_variables_initializer().run()
validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}
test_feed = {x: mnist.test.images, y_: mnist.test.labels}

# 循环的训练神经网络。
for i in range(TRAINING_STEPS):
if i % 1000 == 0:
validate_acc = sess.run(accuracy, feed_dict=validate_feed)
print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))

xs, ys = mnist.train.next_batch(BATCH_SIZE)
sess.run(train_op, feed_dict={x: xs, y_: ys})

test_acc = sess.run(accuracy, feed_dict=test_feed)
print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))

# 4. 主程序入口，这里设定模型训练次数为5000次
def main(argv=None):
train(mnist)

if __name__=='__main__':
main()

'''
After 0 training step(s), validation accuracy using average model is 0.0978
After 1000 training step(s), validation accuracy using average model is 0.9726
After 2000 training step(s), validation accuracy using average model is 0.9808
After 3000 training step(s), validation accuracy using average model is 0.9816
After 4000 training step(s), validation accuracy using average model is 0.9818
After 5000 training step(s), test accuracy using average model is 0.9832
'''