书山有路勤为径，学海无涯苦作舟
1.RNN网络
2.tensorflow构建mnist-RNN网络
2.1导入数据集
import tensorflow as tf
import numpy as np
from tensorflow.contrib import rnn
from tensorflow.examples.tutorials.mnist import input_data
sess = tf.Session()
mnist = input_data.read_data_sets('data', one_hot=True)
print (mnist.train.images.shape)
Extracting data\train-images-idx3-ubyte.gz
Extracting data\train-labels-idx1-ubyte.gz
Extracting data\t10k-images-idx3-ubyte.gz
Extracting data\t10k-labels-idx1-ubyte.gz
(55000, 784)
2.2 设置参数
设定学习率
设定输入大小（将28*28大小的图片按照行切分，分成28份，每份28个像素点）
持续输入28行
指定隐藏层级的个数，LSTM的层数，分类数
指定输入_X（），大小为28*28=784 ，后面还要reshape的操作
指定输出的结果y
指定batch_size,dropout
lr = 1e-3
input_size = 28 # 每行输入28个特征点
timestep_size = 28 # 持续输入28行
hidden_size = 256 # 隐含层的数量
layer_num = 2 # LSTM layer 的层数
class_num = 10 # 10分类问题
_X = tf.placeholder(tf.float32, [None, 784])
y = tf.placeholder(tf.float32, [None, class_num])
batch_size = tf.placeholder(tf.int32, [])
keep_prob = tf.placeholder(tf.float32, [])
2.3 网络结构：
最后连接softmax，多分类的归一化
变换X，转换为28行28列
定义LSTM的基本单元，借助tensorflow的contrib的rnn
指定基本的隐藏层单元数，reuse指定是否要重复利用
加上dropout，设定保留率
2.4构造两层的LSTM的堆叠，借助multirnncell，根据指定层数自动堆叠。
用0值初始化，将初始值都变为0
会得到很多的中间结果，但是我们只想要最后一个结果。用for拿到28行的分别的计算结果
【：，timestep，：】并行计算，“：”会得到一个batch中的所有的第一步的lstm的计算结果，
append到列表。索引得最后的值。
X = tf.reshape(_X, [-1, 28, 28])
def lstm_cell():
cell = rnn.LSTMCell(hidden_size, reuse=tf.get_variable_scope().reuse)
return rnn.DropoutWrapper(cell, output_keep_prob=keep_prob)
mlstm_cell = tf.contrib.rnn.MultiRNNCell([lstm_cell() for _ in range(layer_num)], state_is_tuple = True)
#用全零来初始化状态
init_state = mlstm_cell.zero_state(batch_size, dtype=tf.float32)
#得到每一层的输出结果
outputs = list()
state = init_state
with tf.variable_scope('RNN'):
for timestep in range(timestep_size):
if timestep > 0:
tf.get_variable_scope().reuse_variables()
(cell_output, state) = mlstm_cell(X[:, timestep, :],state)
outputs.append(cell_output)
h_state = outputs[-1]
2.5 迭代计算
拿到最后的结果，再连接一个全连接层，然后将全连接层的结果再连接一个softmax的归一化。
从LSTM拿到的只是一个特征，借助全连接层得到输出结果，借助softmax进行归一化，得到每一个类别的概率。
添加全连接层，构建W，shape是LSTM得到256个特征，10个类别。
构建偏置
做sotfmax的归一化，得到每个类别的预测
计算损失和优化器
初始化，迭代计算
计算测试集的准确率
#Softmax层参数
W = tf.Variable(tf.truncated_normal([hidden_size, class_num], stddev=0.1), dtype=tf.float32)
bias = tf.Variable(tf.constant(0.1,shape=[class_num]), dtype=tf.float32)
y_pre = tf.nn.softmax(tf.matmul(h_state, W) + bias)
# 损失和评估函数
cross_entropy = -tf.reduce_mean(y * tf.log(y_pre))
train_op = tf.train.AdamOptimizer(lr).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_pre,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
sess.run(tf.global_variables_initializer())
for i in range(2000):
_batch_size = 128
batch = mnist.train.next_batch(_batch_size)
if (i+1)%200 == 0:
train_accuracy = sess.run(accuracy, feed_dict={
_X:batch[0], y: batch[1], keep_prob: 1.0, batch_size: _batch_size})
# 已经迭代完成的 epoch 数: mnist.train.epochs_completed
print ("Iter%d, step %d, training accuracy %g" % ( mnist.train.epochs_completed, (i+1), train_accuracy))
sess.run(train_op, feed_dict={_X: batch[0], y: batch[1], keep_prob: 0.5, batch_size: _batch_size})
# 计算测试数据的准确率
print ("test accuracy %g"% sess.run(accuracy, feed_dict={
_X: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0, batch_size:mnist.test.images.shape[0]}))
Iter0, step 200, training accuracy 0.929688
Iter0, step 400, training accuracy 0.96875
Iter1, step 600, training accuracy 0.953125
Iter1, step 800, training accuracy 0.976562
Iter2, step 1000, training accuracy 0.96875
Iter2, step 1200, training accuracy 0.984375
Iter3, step 1400, training accuracy 0.976562
Iter3, step 1600, training accuracy 0.984375
Iter4, step 1800, training accuracy 0.992188
Iter4, step 2000, training accuracy 0.976562
test accuracy 0.9839
2.6 观测每个阶段x1···x28的结果
指定每次输入的图像个数bitchsize
计算得到outouts（28,5,256） 28表示28个结果，5张图像的中间结果，256是得到的特征数目
_batch_size = 5
X_batch, y_batch = mnist.test.next_batch(_batch_size)
print (X_batch.shape, y_batch.shape)
_outputs, _state = sess.run([outputs, state],feed_dict={
_X: X_batch, y: y_batch, keep_prob: 1.0, batch_size: _batch_size})
print ('_outputs.shape =', np.asarray(_outputs).shape)
(5, 784) (5, 10)
_outputs.shape = (28, 5, 256)
import matplotlib.pyplot as plt
print (mnist.train.labels[4])
X3 = mnist.train.images[4]
img3 = X3.reshape([28, 28])
plt.imshow(img3, cmap='gray')
plt.show()
[ 1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
X3.shape = [-1, 784]
y_batch = mnist.train.labels[0]
y_batch.shape = [-1, class_num]
X3_outputs = np.array(sess.run(outputs, feed_dict={
_X: X3, y: y_batch, keep_prob: 1.0, batch_size: 1}))
print (X3_outputs.shape)
X3_outputs.shape = [28, hidden_size]
print (X3_outputs.shape)
(28, 1, 256)
(28, 256)
随机取得一个数字图像，进行迭代，查看每次输入的结果
每次run得到中间结果， 逐个显示outputs的结果
并且 对每个结果的结果都进行softmax的阶段性预测结果。
h_W = sess.run(W, feed_dict={
_X:X3, y: y_batch, keep_prob: 1.0, batch_size: 1})
h_bias = sess.run(bias, feed_dict={
_X:X3, y: y_batch, keep_prob: 1.0, batch_size: 1})
h_bias.shape = [-1, 10]
bar_index = range(class_num)
for i in range(X3_outputs.shape[0]):
plt.subplot(7, 4, i+1)
X3_h_shate = X3_outputs[i, :].reshape([-1, hidden_size])
pro = sess.run(tf.nn.softmax(tf.matmul(X3_h_shate, h_W) + h_bias))
plt.bar(bar_index, pro[0], width=0.2 , align='center')
plt.axis('off')
plt.show()
3. RNN实现时间序列预测分析
3.1构建辅助函数
(1)加载数据函数
- 读取数据
- 标准化；当前值-均值 / 标准差 （数据浮动差异大，预测起来比较困难，所以需要标准化）
(2)划分数据集的函数
(3)可视化
import csv
import numpy as np
import matplotlib.pyplot as plt
def load_series(filename, series_idx=1):
try:
with open(filename) as csvfile:
csvreader = csv.reader(csvfile)
data = [float(row[series_idx]) for row in csvreader if len(row) > 0]
normalized_data = (data - np.mean(data)) / np.std(data)
return normalized_data
except IOError:
return None
def split_data(data, percent_train=0.80):
num_rows = len(data)
train_data, test_data = [], []
for idx, row in enumerate(data):
if idx < num_rows * percent_train:
train_data.append(row)
else:
test_data.append(row)
return train_data, test_data
if __name__=='__main__':
timeseries = load_series('international-airline-passengers.csv')
print(np.shape(timeseries))
plt.figure()
plt.plot(timeseries)
plt.show()
3.2 构建RNN网络时间预测
指定seq_size=5， 指定当前片段大小， 基于5个小片段预测下一个。
构建超参数，输入，size，隐藏层数，w，b，x，y
指定损失，训练，保存模型。
时间序列构建训练集和测试集
构建训练集的数据的标签需要和测试集错开一个时间，用前面的数据预测后面的值。
第0的数据预测第1的数据，用第N个的数据预测第N+1的数据。
比如训练数据集取得0-5的位置，标签的值就要取得1-6的位置，这样才能实现前一个预测下一个。
指定迭代的停止条件
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import data_loader
import matplotlib.pyplot as plt
class SeriesPredictor:
def __init__(self, input_dim, seq_size, hidden_dim):
# Hyperparameters
self.input_dim = input_dim
self.seq_size = seq_size
self.hidden_dim = hidden_dim
# Weight variables and input placeholders
self.W_out = tf.Variable(tf.random_normal([hidden_dim, 1]), name='W_out')
self.b_out = tf.Variable(tf.random_normal([1]), name='b_out')
self.x = tf.placeholder(tf.float32, [None, seq_size, input_dim])
self.y = tf.placeholder(tf.float32, [None, seq_size])
# Cost optimizer
self.cost = tf.reduce_mean(tf.square(self.model() - self.y))
self.train_op = tf.train.AdamOptimizer(learning_rate=0.01).minimize(self.cost)
# Auxiliary ops
self.saver = tf.train.Saver()
def model(self):
"""
:param x: inputs of size [T, batch_size, input_size]
:param W: matrix of fully-connected output layer weights
:param b: vector of fully-connected output layer biases
"""
cell = rnn.BasicLSTMCell(self.hidden_dim)
outputs, states = tf.nn.dynamic_rnn(cell, self.x, dtype=tf.float32)
num_examples = tf.shape(self.x)[0]
W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1])
out = tf.matmul(outputs, W_repeated) + self.b_out
out = tf.squeeze(out)
return out
def train(self, train_x, train_y, test_x, test_y):
with tf.Session() as sess:
tf.get_variable_scope().reuse_variables()
sess.run(tf.global_variables_initializer())
max_patience = 3
patience = max_patience
min_test_err = float('inf')
step = 0
while patience > 0:
_, train_err = sess.run([self.train_op, self.cost], feed_dict={self.x: train_x, self.y: train_y})
if step % 100 == 0:
test_err = sess.run(self.cost, feed_dict={self.x: test_x, self.y: test_y})
print('step: {}\t\ttrain err: {}\t\ttest err: {}'.format(step, train_err, test_err))
if test_err < min_test_err:
min_test_err = test_err
patience = max_patience
else:
patience -= 1
step += 1
save_path = self.saver.save(sess, './model/')
print('Model saved to {}'.format(save_path))
def test(self, sess, test_x):
tf.get_variable_scope().reuse_variables()
self.saver.restore(sess, './model/')
output = sess.run(self.model(), feed_dict={self.x: test_x})
return output
def plot_results(train_x, predictions, actual, filename):
plt.figure()
num_train = len(train_x)
plt.plot(list(range(num_train)), train_x, color='b', label='training data')
plt.plot(list(range(num_train, num_train + len(predictions))), predictions, color='r', label='predicted')
plt.plot(list(range(num_train, num_train + len(actual))), actual, color='g', label='test data')
plt.legend()
if filename is not None:
plt.savefig(filename)
else:
plt.show()
if __name__ == '__main__':
seq_size = 5
predictor = SeriesPredictor(input_dim=1, seq_size=seq_size, hidden_dim=100)
data = data_loader.load_series('international-airline-passengers.csv')
train_data, actual_vals = data_loader.split_data(data)
train_x, train_y = [], []
for i in range(len(train_data) - seq_size - 1):
train_x.append(np.expand_dims(train_data[i:i+seq_size], axis=1).tolist())
train_y.append(train_data[i+1:i+seq_size+1])
test_x, test_y = [], []
for i in range(len(actual_vals) - seq_size - 1):
test_x.append(np.expand_dims(actual_vals[i:i+seq_size], axis=1).tolist())
test_y.append(actual_vals[i+1:i+seq_size+1])
predictor.train(train_x, train_y, test_x, test_y)
with tf.Session() as sess:
predicted_vals = predictor.test(sess, test_x)[:,0]
print('predicted_vals', np.shape(predicted_vals))
plot_results(train_data, predicted_vals, actual_vals, 'predictions.png')
prev_seq = train_x[-1]
predicted_vals = []
for i in range(20):
next_seq = predictor.test(sess, [prev_seq])
predicted_vals.append(next_seq[-1])
prev_seq = np.vstack((prev_seq[1:], next_seq[-1]))
plot_results(train_data, predicted_vals, actual_vals, 'hallucinations.png')
4. RNN小demo
# coding=gbk
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
class SeriesPredictor:
def __init__(self, input_dim, seq_size, hidden_dim=10):
# Hyperparameters
self.input_dim = input_dim
self.seq_size = seq_size
self.hidden_dim = hidden_dim
# Weight variables and input placeholders
self.W_out = tf.Variable(tf.random_normal([hidden_dim, 1]), name='W_out')
self.b_out = tf.Variable(tf.random_normal([1]), name='b_out')
self.x = tf.placeholder(tf.float32, [None, seq_size, input_dim])
self.y = tf.placeholder(tf.float32, [None, seq_size])
# Cost optimizer
self.cost = tf.reduce_mean(tf.square(self.model() - self.y))
self.train_op = tf.train.AdamOptimizer().minimize(self.cost)
# Auxiliary ops
self.saver = tf.train.Saver()
def model(self):
"""
:param x: inputs of size [T, batch_size, input_size]
:param W: matrix of fully-connected output layer weights
:param b: vector of fully-connected output layer biases
"""
cell = rnn.BasicLSTMCell(self.hidden_dim)
outputs, states = tf.nn.dynamic_rnn(cell, self.x, dtype=tf.float32)
num_examples = tf.shape(self.x)[0]
# tf.expand_dims,增加一个新的维度
#W_repeated = tf.tile(tf.expand_dims(self.W_out, 0), [num_examples, 1, 1])
tf_expand = tf.expand_dims(self.W_out, 0)
tf_tile = tf.tile(tf_expand, [num_examples, 1, 1])
out = tf.matmul(outputs, tf_tile) + self.b_out
# tf.squeeze 删除所有维度是1的
out = tf.squeeze(out)
return out
def train(self, train_x, train_y):
with tf.Session() as sess:
tf.get_variable_scope().reuse_variables()
sess.run(tf.global_variables_initializer())
for i in range(1000):
_, mse = sess.run([self.train_op, self.cost], feed_dict={self.x: train_x, self.y: train_y})
if i % 100 == 0:
print(i, mse)
save_path = self.saver.save(sess, './model')
print('Model saved to {}'.format(save_path))
def test(self, test_x):
with tf.Session() as sess:
tf.get_variable_scope().reuse_variables()
self.saver.restore(sess, './model')
output = sess.run(self.model(), feed_dict={self.x: test_x})
return output
if __name__ == '__main__':
predictor = SeriesPredictor(input_dim=1, seq_size=4, hidden_dim=10)
train_x = [[[1], [2], [5], [6]],
[[5], [7], [7], [8]],
[[3], [4], [5], [7]]]
train_y = [[1, 3, 7, 11],
[5, 12, 14, 15],
[3, 7, 9, 12]]
predictor.train(train_x, train_y)
test_x = [[[1], [2], [3], [4]], # 1, 3, 5, 7
[[4], [5], [6], [7]]] # 4, 9, 11, 13
actual_y = [[[1], [3], [5], [7]],
[[4], [9], [11], [13]]]
pred_y = predictor.test(test_x)
print("\nLets run some tests!\n")
for i, x in enumerate(test_x):
print("When the input is {}".format(x))
print("The ground truth output should be {}".format(actual_y[i]))
print("And the model thinks it is {}\n".format(pred_y[i]))
2311
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
