tensorflow优化器比较

TensorFlow中提供的Optimizer包括：GradientDescentOptimizer，AdagradOptimizer，AdadeltaOptimizer，MomentumOptimizer ，AdamOptimizer ，FtrlOptimizer ，RMSPropOptimizer

本文主要比较这几种优化器在正态分布下的优化效果，由于各个算法都是为解决某一特定问题产生的，因此在这个训练集上的表现并不能作为算法间优劣评判的标准，测试用例如下：

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
# Create dataset, 200 points
x_data = np.linspace(-0.5, 0.5, 200)[:,np.newaxis]
noise = np.random.normal(0, 0.02, x_data.shape)
y_data = np.square(x_data) + noise
# Define two placeholders for x and y
x = tf.placeholder(tf.float32, [None, 1])
y = tf.placeholder(tf.float32, [None, 1])
# Define inner-layer of network
Weights_L1 = tf.Variable(tf.random_normal([1, 10]))
biases_L1 = tf.Variable(tf.zeros([1, 10]))
Wx_plus_b_L1 = tf.matmul(x, Weights_L1) + biases_L1
L1 = tf.nn.tanh(Wx_plus_b_L1)
# Define the output-layer of network
Weights_L2 = tf.Variable(tf.random_normal([10, 1]))
biases_L2 = tf.Variable(tf.zeros([1, 10]))
Wx_plus_b_L2 = tf.matmul(L1, Weights_L2) + biases_L2
prediction = tf.nn.tanh(Wx_plus_b_L2)
# Define loss function
loss = tf.reduce_mean(tf.square(y - prediction))
# Define backword propation with GD
train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
# Start workflow
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for i in range(1000):
        sess.run(train_step, feed_dict={x : x_data, y : y_data})
    prediction_value = sess.run(prediction, feed_dict={x : x_data})
plt.figure()
plt.scatter(x_data, y_data)
plt.plot(x_data, prediction_value, 'r-', lw=5)
plt.show()