退化学习率实例，Maxout网络实现MNIST分类

退化学习率实例： 

import tensorflow as tf

global_step = tf.Variable(0, trainable = False)
initial_learning_rate = 0.1  # 初始化学习率
# 退化学习率，tf.train.exponential_decay : 当前迭代到 global_step后， 学习率每一步都按照每10步去缩小到 0.9的速度衰减
learning_rate = tf.train.exponential_decay( initial_learning_rate,
                                            global_step = global_step,
                                            decay_steps = 10, decay_rate = 0.9)
# 优化函数
opt = tf.train.GradientDescentOptimizer(learning_rate)
# global_step 叠加
add_global = global_step.assign_add(1)

with tf.Session() as sess:
    tf.global_variables_initializer().run()
    print(sess.run(learning_rate))

    for i in range(30):
        g, rate = sess.run([add_global, learning_rate])
        print(g, rate)

结果：
# 0.1
# 1 0.1
# 2 0.09895193
# 3 0.09791484
# 4 0.09688862
# 5 0.095873155
# 6 0.094868325
# 7 0.09387404
# 8 0.092890166
# 9 0.09191661
# 10 0.09095325
# 11 0.089999996
# 12 0.08905673
# 13 0.087199755
# 14 0.08628584
# 15 0.08628584
# 16 0.0853815

 

Maxout网络实现MNIST分类：

# 使用 Maxout 网络实现 MNIST 分类
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot = True)
import pylab

tf.reset_default_graph()
# 定义占位符
x = tf.placeholder(tf.float32, [None, 784])  # MNIST 数据集的维度是 28*28 = 784
y = tf.placeholder(tf.float32, [None, 10])

w = tf.Variable(tf.random_normal([784, 10]))
b = tf.Variable(tf.zeros([10]))

z = tf.matmul(x , w) +b
maxout = tf.reduce_max(z ,axis =1, keep_dims = True)
# 设置学习参数
w2 = tf.Variable(tf.truncated_normal([1,10], stddev=0.1))
b2 = tf.Variable(tf.zeros([1]))
# 构建模型 # 损失函数
cost = tf.nn.softmax(tf.matmul(maxout, w2) +b2)
learning_rate = 0.04
# 梯度下降优化器
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
trianing_epochs = 25
batch_size = 100
display_step = 1

# 启动 session
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())  # 初始化 OP
    # 启动循环开始训练
    for epoch in range(trianing_epochs):
        avg_cost = 0.
        total_batch = int(mnist.train.num_examples / batch_size)
        # 循环所有数据集
        for i in range(total_batch):
            batch_xs , batch_ys = mnist.train.next_batch(batch_size)
            # 运行优化器
            _, c = sess.run([optimizer, cost], feed_dict = {x: batch_xs, y:batch_ys})
            # 计算平均 loss 值
            avg_cost += c / total_batch
        # 显示训练中的详细信息
        if (epoch+1) % display_step == 0:
            print('epoch = ' , '%s' %(epoch+1))
            print( 'cost= ' , '%s' %avg_cost)
    print("Finished !")