python-tensorflow框架学习 -3

本文接着  python-tensorflow框架学习 -2，本部分后面会持续更新，目前只写了两个子部份: CNN + RNN

part three： CNN - mnist classification;  RNN - mnist classification ; RNN - mnist regression

* CNN - mnist classification ： 利用CNN网络构建mnist数据集的分类任务

* RNN - mnist classification ： 利用RNN网络构建mnist数据集的分类任务

* RNN - cos function regression ： 利用RNN网络构建cos函数预测回归任务

3.1 CNN -classification.py

# 一个简单的mnist手写数字识别分类任务(CNN网络) 600step: 86%
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
tf.set_random_seed(1)
np.random.seed(1)
BATCH_SIZE = 50
LR = 0.001

# 1 构建测试集、并为训练集设置占位符 【mnist数据集，如果本地相应数据集下没有则会通过滤镜进行下载，否则直接使用】
mnist = input_data.read_data_sets('./mnist', one_hot=True)   # they has been normalized to range (0,1)
test_x = mnist.test.images[:2000]
test_y = mnist.test.labels[:2000]        # 先从mnist的test数据集中获取相应部分的测试集

tf_x = tf.placeholder(tf.float32, [None, 28*28])/225.
image = tf.reshape(tf_x, [-1, 28, 28, 1])     # [batch, height, width, channel] :[batch, 28, 28, 1]
tf_y = tf.placeholder(tf.int32, [None, 10])

# 2 CNN网络构建
conv1 = tf.layers.conv2d(inputs=image, filters=16, kernel_size=5
                         ,strides=1, padding='same', activation=tf.nn.relu
                         )     # [batch, 28, 28, 1] to [batch, 28, 28, 16]
pool1 = tf.layers.max_pooling2d(conv1, pool_size=2, strides=2)     # to [batch, 14, 14, 16]
conv2 = tf.layers.conv2d(inputs=pool1, filters=32, kernel_size=5, strides=1, padding='same', activation=tf.nn.relu)  # to [batch, 14, 14, 32]
pool2 = tf.layers.max_pooling2d(conv2, pool_size=2, strides=2)  # to [batch, 7, 7, 32]
flat_res = tf.reshape(pool2, [-1, 7*7*32])      # to [batch, 7*7*32]
output = tf.layers.dense(flat_res, 10)          # output layer: to [batch, 10]

# 3 构建相应的loss和train_op
loss = tf.losses.softmax_cross_entropy(onehot_labels=tf_y, logits=output)    # cross entropy func
train_op = tf.train.AdamOptimizer(LR).minimize(loss)
accuracy = tf.metrics.accuracy(labels=tf.argmax(tf_y, axis=1), predictions=tf.argmax(output,axis=1),)[1]  # lables和preds均是一个10维度的向量
sess = tf.Session()
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
sess.run(init_op)    # initialize var in graph

# 4 following function (plot_with_labels) is for visualization, can be ignored if not interested 【TSNE 降维方法】
from matplotlib import cm
try:
    from sklearn.manifold import TSNE; HAS_SK = True
except:
    HAS_SK = False; print('\nplease install sklearn for layer visualization')
def plot_with_labels(lowDWeights, labels):
    plt.cla()
    X, Y = lowDWeights[:, 0], lowDWeights[:, 1]
    for x, y, s in zip(X, Y, labels):
        c = cm.rainbow(int(225*s/9))
        plt.text(x, y, s, backgroundcolor=c, fontsize=9)
    plt.xlim(X.min(),X.max())
    plt.ylim(Y.min(), Y.max())
    plt.title('Visualize last layer')
    plt.show()
    plt.pause(0.01)
plt.ion()

# 5 训练
for step in range(600):
    b_x, b_y = mnist.train.next_batch(BATCH_SIZE)
    _, loss_ = sess.run([train_op, loss], feed_dict={tf_x:b_x, tf_y:b_y})
    if step % 50 == 0:
        accuracy_, flat_representation = sess.run([accuracy, flat_res], {tf_x: test_x, tf_y: test_y})
        print('step:', step,"| train loss :%.4f"%loss_, "|test accuracy:%.4f"%accuracy_)

        if HAS_SK:
            # visualization of trained flatten layer (T-SNE)
            tsne = TSNE(perplexity=30, n_components=2, init='pca',n_iter=5000)
            plot_only = 500
            low_dim_embs = tsne.fit_transform(flat_representation[:plot_only, :])
            labels = np.argmax(test_y,axis=1)[:plot_only]
            plot_with_labels(low_dim_embs, labels)
plt.ioff()

# 6 打印10个预测和真实的例子
test_output = sess.run(output, feed_dict={tf_x: test_x[:10]})
pred_y = np.argmax(test_output, 1)
print('pred_numbers:', pred_y)
print('real_numbers:', np.argmax(test_y[:10], 1))

结果分析：每过50step，会打印相应的loss 和 accuracy value，并show出当前tsne降维后的各类分类结果。[tsne是一个动态变化的过程，便于分析， 本博客仅仅展示600step的最终结果]

step: 0 | train loss :2.2956 |test accuracy:0.1365
step: 50 | train loss :0.3620 |test accuracy:0.4765
step: 100 | train loss :0.2210 |test accuracy:0.6137
step: 150 | train loss :0.2453 |test accuracy:0.6873
step: 200 | train loss :0.1987 |test accuracy:0.7373
step: 250 | train loss :0.2205 |test accuracy:0.7717
step: 300 | train loss :0.0810 |test accuracy:0.7940
step: 350 | train loss :0.0702 |test accuracy:0.8144
step: 400 | train loss :0.1085 |test accuracy:0.8296
step: 450 | train loss :0.0775 |test accuracy:0.8425
step: 500 | train loss :0.1935 |test accuracy:0.8535
step: 550 | train loss :0.1200 |test accuracy:0.8618
pred_numbers: [7 2 1 0 4 1 4 9 5 9]
real_numbers: [7 2 1 0 4 1 4 9 5 9]

分类结果tsne可视化最终状态：

3.2 RNN -classification.py 【同样的分类数据集，利用RNN网络来建立分类任务】

# mnist的一个简单的rnn分类问题示例代码 89% 的准确率
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import numpy as np
import matplotlib.pyplot as plt

tf.set_random_seed(1)
np.random.seed(1)

# Hyper Parameters
BATCH_SIZE = 64
TIME_STEP = 28          # rnn time step / image height
INPUT_SIZE = 28         # rnn input size / image width
LR = 0.01               # learning rate

# data
mnist = input_data.read_data_sets('./mnist', one_hot=True)              # they has been normalized to range (0,1)
test_x = mnist.test.images[:2000]
test_y = mnist.test.labels[:2000]

# plot one example
print(mnist.train.images.shape)     # (55000, 28 * 28)
print(mnist.train.labels.shape)   # (55000, 10)
plt.imshow(mnist.train.images[0].reshape((28, 28)), cmap='gray')
plt.title('%i' % np.argmax(mnist.train.labels[0]))
plt.show()

# tensorflow placeholders
tf_x = tf.placeholder(tf.float32, [None, TIME_STEP * INPUT_SIZE])       # shape(batch, 784)
image = tf.reshape(tf_x, [-1, TIME_STEP, INPUT_SIZE])                   # (batch, height, width, channel)
tf_y = tf.placeholder(tf.int32, [None, 10])                             # input y

# RNN
rnn_cell = tf.nn.rnn_cell.LSTMCell(num_units=64)
outputs, (h_c, h_n) = tf.nn.dynamic_rnn(
    rnn_cell,                   # cell you have chosen
    image,                      # input
    initial_state=None,         # the initial hidden state
    dtype=tf.float32,           # must given if set initial_state = None
    time_major=False,           # False: (batch, time step, input); True: (time step, batch, input)
)
output = tf.layers.dense(outputs[:, -1, :], 10)              # output based on the last output step

loss = tf.losses.softmax_cross_entropy(onehot_labels=tf_y, logits=output)           # compute cost
train_op = tf.train.AdamOptimizer(LR).minimize(loss)

accuracy = tf.metrics.accuracy(          # return (acc, update_op), and create 2 local variables
    labels=tf.argmax(tf_y, axis=1), predictions=tf.argmax(output, axis=1),)[1]

sess = tf.Session()
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) # the local var is for accuracy_op
sess.run(init_op)     # initialize var in graph

for step in range(1200):    # training
    b_x, b_y = mnist.train.next_batch(BATCH_SIZE)
    _, loss_ = sess.run([train_op, loss], {tf_x: b_x, tf_y: b_y})
    if step % 50 == 0:      # testing
        accuracy_ = sess.run(accuracy, {tf_x: test_x, tf_y: test_y})
        print('train loss: %.4f' % loss_, '| test accuracy: %.2f' % accuracy_)

# print 10 predictions from test data
test_output = sess.run(output, {tf_x: test_x[:10]})
pred_y = np.argmax(test_output, 1)
print(pred_y, 'prediction number')
print(np.argmax(test_y[:10], 1), 'real number')

结果：

示例手写数字图例：

train loss: 2.3271 | test accuracy: 0.14
train loss: 0.8738 | test accuracy: 0.42
train loss: 0.5579 | test accuracy: 0.53
train loss: 0.5082 | test accuracy: 0.62
train loss: 0.3069 | test accuracy: 0.68
train loss: 0.1639 | test accuracy: 0.72
train loss: 0.2474 | test accuracy: 0.75
train loss: 0.1535 | test accuracy: 0.77
train loss: 0.0769 | test accuracy: 0.79
train loss: 0.2508 | test accuracy: 0.80
train loss: 0.1221 | test accuracy: 0.82
train loss: 0.3389 | test accuracy: 0.83
train loss: 0.1557 | test accuracy: 0.84
train loss: 0.0876 | test accuracy: 0.84
train loss: 0.1594 | test accuracy: 0.85
train loss: 0.2142 | test accuracy: 0.86
train loss: 0.0384 | test accuracy: 0.86
train loss: 0.2602 | test accuracy: 0.87
train loss: 0.2522 | test accuracy: 0.87
train loss: 0.0524 | test accuracy: 0.88
train loss: 0.1400 | test accuracy: 0.88
train loss: 0.0989 | test accuracy: 0.88
train loss: 0.1122 | test accuracy: 0.89
train loss: 0.0807 | test accuracy: 0.89
[7 2 1 0 4 1 4 9 5 9] prediction number
[7 2 1 0 4 1 4 9 5 9] real number

3.3 RNN -regression.py 【基于RNN网络，构建一个预测cos函数的回归模型】

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

# Hyper Parameters
TIME_STEP = 10       # rnn time step
INPUT_SIZE = 1      # rnn input size
CELL_SIZE = 32      # rnn cell size
LR = 0.02           # learning rate

# show data
steps = np.linspace(0, np.pi*2, 100, dtype=np.float32)
x_np = np.sin(steps); y_np = np.cos(steps)    # float32 for converting torch FloatTensor
plt.plot(steps, y_np, 'r-', label='target (cos)'); plt.plot(steps, x_np, 'b-', label='input (sin)')
plt.legend(loc='best'); plt.show()

# tensorflow placeholders
tf_x = tf.placeholder(tf.float32, [None, TIME_STEP, INPUT_SIZE])        # shape(batch, 5, 1)
tf_y = tf.placeholder(tf.float32, [None, TIME_STEP, INPUT_SIZE])          # input y

# RNN
rnn_cell = tf.nn.rnn_cell.LSTMCell(num_units=CELL_SIZE)
init_s = rnn_cell.zero_state(batch_size=1, dtype=tf.float32)    # very first hidden state
outputs, final_s = tf.nn.dynamic_rnn(
    rnn_cell,                   # cell you have chosen
    tf_x,                       # input
    initial_state=init_s,       # the initial hidden state
    time_major=False,           # False: (batch, time step, input); True: (time step, batch, input)
)
outs2D = tf.reshape(outputs, [-1, CELL_SIZE])                       # reshape 3D output to 2D for fully connected layer
net_outs2D = tf.layers.dense(outs2D, INPUT_SIZE)
outs = tf.reshape(net_outs2D, [-1, TIME_STEP, INPUT_SIZE])          # reshape back to 3D

loss = tf.losses.mean_squared_error(labels=tf_y, predictions=outs)  # compute cost
train_op = tf.train.AdamOptimizer(LR).minimize(loss)

sess = tf.Session()
sess.run(tf.global_variables_initializer())     # initialize var in graph

plt.figure(1, figsize=(12, 5)); plt.ion()       # continuously plot

for step in range(60):
    start, end = step * np.pi, (step+1)*np.pi   # time range
    # use sin predicts cos
    steps = np.linspace(start, end, TIME_STEP)
    x = np.sin(steps)[np.newaxis, :, np.newaxis]    # shape (batch, time_step, input_size)
    y = np.cos(steps)[np.newaxis, :, np.newaxis]
    if 'final_s_' not in globals():                 # first state, no any hidden state
        feed_dict = {tf_x: x, tf_y: y}
    else:                                           # has hidden state, so pass it to rnn
        feed_dict = {tf_x: x, tf_y: y, init_s: final_s_}
    _, pred_, final_s_ = sess.run([train_op, outs, final_s], feed_dict)     # train

    # plotting
    plt.plot(steps, y.flatten(), 'r-'); plt.plot(steps, pred_.flatten(), 'b-')
    plt.ylim((-1.2, 1.2)); plt.draw(); plt.pause(0.05)

plt.ioff(); plt.show()

真实曲线：红色曲线：cos函数(y)   +  蓝色曲线：sin函数(x)  ，goal: 基于sin函数预测cos函数。

预测;