实例55：RNN
# -*- coding: utf-8 -*-
# 1. 定义基本函数
import copy, numpy as np
np.random.seed(0) #随机数生成器的种子，可以每次得到一样的值
# compute sigmoid nonlinearity
def sigmoid(x): #激活函数
output = 1/(1+np.exp(-x))
return output
# convert output of sigmoid function to its derivative
def sigmoid_output_to_derivative(output):#激活函数的导数
return output*(1-output)
# 2. 建立二进制映射
int2binary = {} #整数到其二进制表示的映射
binary_dim = 8 #暂时制作256以内的减法
## 计算0-256的二进制表示
largest_number = pow(2,binary_dim)
binary = np.unpackbits(
np.array([range(largest_number)],dtype=np.uint8).T,axis=1)
for i in range(largest_number):
int2binary[i] = binary[i]
# 3. 定义参数
# input variables
alpha = 0.9 #学习速率
input_dim = 2 #输入的维度是2
hidden_dim = 16
output_dim = 1 #输出维度为1
# initialize neural network weights
synapse_0 = (2*np.random.random((input_dim,hidden_dim)) - 1)*0.05 #维度为2*16， 2是输入维度，16是隐藏层维度
synapse_1 = (2*np.random.random((hidden_dim,output_dim)) - 1)*0.05
synapse_h = (2*np.random.random((hidden_dim,hidden_dim)) - 1)*0.05
# => [-0.05, 0.05)，
# 用于存放反向传播的权重更新值
synapse_0_update = np.zeros_like(synapse_0)
synapse_1_update = np.zeros_like(synapse_1)
synapse_h_update = np.zeros_like(synapse_h)
# 4. 准备样本数据
# training
for j in range(10000):
#生成一个数字a
a_int = np.random.randint(largest_number)
#生成一个数字b,b的最大值取的是largest_number/2,作为被减数，让它小一点。
b_int = np.random.randint(largest_number/2)
#如果生成的b大了，那么交换一下
if a_int<b_int:
tt = a_int
b_int = a_int
a_int=tt
a = int2binary[a_int] # binary encoding
b = int2binary[b_int] # binary encoding
# true answer
c_int = a_int - b_int
c = int2binary[c_int]
# 5. 模型初始化
# 存储神经网络的预测值
d = np.zeros_like(c)
overallError = 0 #每次把总误差清零
layer_2_deltas = list() #存储每个时间点输出层的误差
layer_1_values = list() #存储每个时间点隐藏层的值
layer_1_values.append(np.ones(hidden_dim)*0.1) # 一开始没有隐藏层，所以初始化一下原始值为0.1
# 6. 正向传播
# moving along the positions in the binary encoding
for position in range(binary_dim):#循环遍历每一个二进制位
# generate input and output
X = np.array([[a[binary_dim - position - 1],b[binary_dim - position - 1]]])#从右到左，每次去两个输入数字的一个bit位
y = np.array([[c[binary_dim - position - 1]]]).T#正确答案
# hidden layer (input ~+ prev_hidden)
layer_1 = sigmoid(np.dot(X,synapse_0) + np.dot(layer_1_values[-1],synapse_h))#（输入层 + 之前的隐藏层） -> 新的隐藏层，这是体现循环神经网络的最核心的地方！！！
# output layer (new binary representation)
layer_2 = sigmoid(np.dot(layer_1,synapse_1)) #隐藏层 * 隐藏层到输出层的转化矩阵synapse_1 -> 输出层
layer_2_error = y - layer_2 #预测误差
layer_2_deltas.append((layer_2_error)*sigmoid_output_to_derivative(layer_2)) #把每一个时间点的误差导数都记录下来
overallError += np.abs(layer_2_error[0])#总误差
d[binary_dim - position - 1] = np.round(layer_2[0][0]) #记录下每一个预测bit位
# store hidden layer so we can use it in the next timestep
layer_1_values.append(copy.deepcopy(layer_1))#记录下隐藏层的值，在下一个时间点用
future_layer_1_delta = np.zeros(hidden_dim)
# 7. 反向传播
#反向传播，从最后一个时间点到第一个时间点
for position in range(binary_dim):
X = np.array([[a[position],b[position]]]) #最后一次的两个输入
layer_1 = layer_1_values[-position-1] #当前时间点的隐藏层
prev_layer_1 = layer_1_values[-position-2] #前一个时间点的隐藏层
# error at output layer
layer_2_delta = layer_2_deltas[-position-1] #当前时间点输出层导数
# error at hidden layer
# 通过后一个时间点（因为是反向传播）的隐藏层误差和当前时间点的输出层误差，计算当前时间点的隐藏层误差
layer_1_delta = (future_layer_1_delta.dot(synapse_h.T) + layer_2_delta.dot(synapse_1.T)) * sigmoid_output_to_derivative(layer_1)
# 等到完成了所有反向传播误差计算， 才会更新权重矩阵，先暂时把更新矩阵存起来。
synapse_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)
synapse_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)
synapse_0_update += X.T.dot(layer_1_delta)
future_layer_1_delta = layer_1_delta
# 完成所有反向传播之后，更新权重矩阵。并把矩阵变量清零
synapse_0 += synapse_0_update * alpha
synapse_1 += synapse_1_update * alpha
synapse_h += synapse_h_update * alpha
synapse_0_update *= 0
synapse_1_update *= 0
synapse_h_update *= 0
# 8. 输出结果
# print out progress
if(j % 800 == 0):
#print(synapse_0,synapse_h,synapse_1)
print("总误差:" + str(overallError))
print("Pred:" + str(d))
print("True:" + str(c))
out = 0
for index,x in enumerate(reversed(d)):
out += x*pow(2,index)
print(str(a_int) + " - " + str(b_int) + " = " + str(out))
print("------------")
实例56：使用RNN网络拟合回声信号序列
# -*- coding: utf-8 -*-
# 1. 定义参数生成样本数据
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
num_epochs = 5
total_series_length = 50000
truncated_backprop_length = 15
state_size = 4
num_classes = 2
echo_step = 3
batch_size = 5
num_batches = total_series_length//batch_size//truncated_backprop_length
def generateData():
x = np.array(np.random.choice(2, total_series_length, p=[0.5, 0.5]))#在0 和1 中选择total_series_length个数
y = np.roll(x, echo_step)#向右循环移位【1111000】---【0001111】
y[0:echo_step] = 0
x = x.reshape((batch_size, -1)) # 5,10000
y = y.reshape((batch_size, -1))
return (x, y)
# 2. 定义占位符处理输入数据
batchX_placeholder = tf.placeholder(tf.float32, [batch_size, truncated_backprop_length])
batchY_placeholder = tf.placeholder(tf.int32, [batch_size, truncated_backprop_length])
init_state = tf.placeholder(tf.float32, [batch_size, state_size])
# Unpack columns
inputs_series = tf.unstack(batchX_placeholder, axis=1)#truncated_backprop_length个序列
labels_series = tf.unstack(batchY_placeholder, axis=1)
# 3. 定义网络结构
current_state = init_state
predictions_series = []
losses =[]
for current_input, labels in zip(inputs_series,labels_series):
#for current_input in inputs_series:
current_input = tf.reshape(current_input, [batch_size, 1])
input_and_state_concatenated = tf.concat([current_input, current_state],1) # current_state 4 +1
next_state = tf.contrib.layers.fully_connected(input_and_state_concatenated,state_size
,activation_fn=tf.tanh)
current_state = next_state
logits =tf.contrib.layers.fully_connected(next_state,num_classes,activation_fn=None)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels,logits=logits)
losses.append(loss)
predictions = tf.nn.softmax(logits)
predictions_series.append(predictions)
total_loss = tf.reduce_mean(losses)
train_step = tf.train.AdagradOptimizer(0.3).minimize(total_loss)
# plot 函数
def plot(loss_list, predictions_series, batchX, batchY):
plt.subplot(2, 3, 1)
plt.cla()
plt.plot(loss_list)
for batch_series_idx in range(batch_size):
one_hot_output_series = np.array(predictions_series)[:, batch_series_idx, :]
single_output_series = np.array([(1 if out[0] < 0.5 else 0) for out in one_hot_output_series])
plt.subplot(2, 3, batch_series_idx + 2)
plt.cla()
plt.axis([0, truncated_backprop_length, 0, 2])
left_offset = range(truncated_backprop_length)
left_offset2 = range(echo_step,truncated_backprop_length+echo_step)
label1 = "past values"
label2 = "True echo values"
label3 = "Predictions"
plt.plot(left_offset2, batchX[batch_series_idx, :]*0.2+1.5, "o--b", label=label1)
plt.plot(left_offset, batchY[batch_series_idx, :]*0.2+0.8,"x--b", label=label2)
plt.plot(left_offset, single_output_series*0.2+0.1 , "o--y", label=label3)
plt.legend(loc='best')
plt.draw()
plt.pause(0.0001)
# 4. 建立session训练数据
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
plt.ion()
plt.figure()
plt.show()
loss_list = []
for epoch_idx in range(num_epochs):
x,y = generateData()
_current_state = np.zeros((batch_size, state_size))
print("New data, epoch", epoch_idx)
for batch_idx in range(num_batches):#50000/ 5 /15=分成多少段
start_idx = batch_idx * truncated_backprop_length
end_idx = start_idx + truncated_backprop_length
batchX = x[:,start_idx:end_idx]
batchY = y[:,start_idx:end_idx]
_total_loss, _train_step, _current_state, _predictions_series = sess.run(
[total_loss, train_step, current_state, predictions_series],
feed_dict={
batchX_placeholder:batchX,
batchY_placeholder:batchY,
init_state:_current_state
})
loss_list.append(_total_loss)
# 5. 测试模型及可视化
if batch_idx%100 == 0:
print("Step",batch_idx, "Loss", _total_loss)
plot(loss_list, _predictions_series, batchX, batchY)
plt.ioff()
plt.show()
实例57：构造LSTM对MNIST分类(BasicCell & LSTMCell)
# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
#outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.100226, Training Accuracy= 0.35938
Iter 2560, Minibatch Loss= 1.727162, Training Accuracy= 0.46875
Iter 3840, Minibatch Loss= 1.655166, Training Accuracy= 0.41406
Iter 5120, Minibatch Loss= 1.134178, Training Accuracy= 0.60938
Iter 6400, Minibatch Loss= 1.077124, Training Accuracy= 0.70312
Iter 7680, Minibatch Loss= 0.919439, Training Accuracy= 0.74219
Iter 8960, Minibatch Loss= 0.661384, Training Accuracy= 0.77344
Iter 10240, Minibatch Loss= 0.611414, Training Accuracy= 0.79688
Iter 11520, Minibatch Loss= 0.649199, Training Accuracy= 0.77344
Iter 12800, Minibatch Loss= 0.732224, Training Accuracy= 0.75781
Iter 14080, Minibatch Loss= 0.520225, Training Accuracy= 0.84375
Iter 15360, Minibatch Loss= 0.435234, Training Accuracy= 0.86719
Iter 16640, Minibatch Loss= 0.354433, Training Accuracy= 0.91406
Iter 17920, Minibatch Loss= 0.391198, Training Accuracy= 0.85938
Iter 19200, Minibatch Loss= 0.574429, Training Accuracy= 0.80469
Iter 20480, Minibatch Loss= 0.508154, Training Accuracy= 0.83594
Iter 21760, Minibatch Loss= 0.281986, Training Accuracy= 0.91406
Iter 23040, Minibatch Loss= 0.427695, Training Accuracy= 0.86719
Iter 24320, Minibatch Loss= 0.286359, Training Accuracy= 0.92969
Iter 25600, Minibatch Loss= 0.278457, Training Accuracy= 0.92188
Iter 26880, Minibatch Loss= 0.228263, Training Accuracy= 0.95312
Iter 28160, Minibatch Loss= 0.280953, Training Accuracy= 0.90625
Iter 29440, Minibatch Loss= 0.234286, Training Accuracy= 0.93750
Iter 30720, Minibatch Loss= 0.310759, Training Accuracy= 0.89062
Iter 32000, Minibatch Loss= 0.365036, Training Accuracy= 0.91406
Iter 33280, Minibatch Loss= 0.264047, Training Accuracy= 0.93750
Iter 34560, Minibatch Loss= 0.290281, Training Accuracy= 0.90625
Iter 35840, Minibatch Loss= 0.196424, Training Accuracy= 0.92969
Iter 37120, Minibatch Loss= 0.198498, Training Accuracy= 0.94531
Iter 38400, Minibatch Loss= 0.342719, Training Accuracy= 0.89844
Iter 39680, Minibatch Loss= 0.319853, Training Accuracy= 0.90625
Iter 40960, Minibatch Loss= 0.209966, Training Accuracy= 0.94531
Iter 42240, Minibatch Loss= 0.109240, Training Accuracy= 0.96875
Iter 43520, Minibatch Loss= 0.245075, Training Accuracy= 0.92188
Iter 44800, Minibatch Loss= 0.169283, Training Accuracy= 0.95312
Iter 46080, Minibatch Loss= 0.141722, Training Accuracy= 0.94531
Iter 47360, Minibatch Loss= 0.139368, Training Accuracy= 0.96875
Iter 48640, Minibatch Loss= 0.129791, Training Accuracy= 0.95312
Iter 49920, Minibatch Loss= 0.135869, Training Accuracy= 0.95312
Iter 51200, Minibatch Loss= 0.167792, Training Accuracy= 0.92969
Iter 52480, Minibatch Loss= 0.219980, Training Accuracy= 0.94531
Iter 53760, Minibatch Loss= 0.253420, Training Accuracy= 0.89844
Iter 55040, Minibatch Loss= 0.310215, Training Accuracy= 0.91406
Iter 56320, Minibatch Loss= 0.140659, Training Accuracy= 0.94531
Iter 57600, Minibatch Loss= 0.147263, Training Accuracy= 0.96094
Iter 58880, Minibatch Loss= 0.145595, Training Accuracy= 0.96875
Iter 60160, Minibatch Loss= 0.140386, Training Accuracy= 0.94531
Iter 61440, Minibatch Loss= 0.112856, Training Accuracy= 0.96094
Iter 62720, Minibatch Loss= 0.066107, Training Accuracy= 0.98438
Iter 64000, Minibatch Loss= 0.141188, Training Accuracy= 0.94531
Iter 65280, Minibatch Loss= 0.124689, Training Accuracy= 0.97656
Iter 66560, Minibatch Loss= 0.136360, Training Accuracy= 0.96094
Iter 67840, Minibatch Loss= 0.127406, Training Accuracy= 0.94531
Iter 69120, Minibatch Loss= 0.190868, Training Accuracy= 0.94531
Iter 70400, Minibatch Loss= 0.146545, Training Accuracy= 0.94531
Iter 71680, Minibatch Loss= 0.144414, Training Accuracy= 0.95312
Iter 72960, Minibatch Loss= 0.139988, Training Accuracy= 0.93750
Iter 74240, Minibatch Loss= 0.078669, Training Accuracy= 0.97656
Iter 75520, Minibatch Loss= 0.194032, Training Accuracy= 0.93750
Iter 76800, Minibatch Loss= 0.123782, Training Accuracy= 0.96875
Iter 78080, Minibatch Loss= 0.153625, Training Accuracy= 0.96094
Iter 79360, Minibatch Loss= 0.125683, Training Accuracy= 0.97656
Iter 80640, Minibatch Loss= 0.146972, Training Accuracy= 0.93750
Iter 81920, Minibatch Loss= 0.175487, Training Accuracy= 0.96094
Iter 83200, Minibatch Loss= 0.078552, Training Accuracy= 0.98438
Iter 84480, Minibatch Loss= 0.136101, Training Accuracy= 0.95312
Iter 85760, Minibatch Loss= 0.141502, Training Accuracy= 0.95312
Iter 87040, Minibatch Loss= 0.068094, Training Accuracy= 0.98438
Iter 88320, Minibatch Loss= 0.289425, Training Accuracy= 0.89844
Iter 89600, Minibatch Loss= 0.127497, Training Accuracy= 0.95312
Iter 90880, Minibatch Loss= 0.090173, Training Accuracy= 0.97656
Iter 92160, Minibatch Loss= 0.135557, Training Accuracy= 0.95312
Iter 93440, Minibatch Loss= 0.122439, Training Accuracy= 0.96094
Iter 94720, Minibatch Loss= 0.171625, Training Accuracy= 0.94531
Iter 96000, Minibatch Loss= 0.140195, Training Accuracy= 0.94531
Iter 97280, Minibatch Loss= 0.156978, Training Accuracy= 0.96094
Iter 98560, Minibatch Loss= 0.050950, Training Accuracy= 0.99219
Iter 99840, Minibatch Loss= 0.086917, Training Accuracy= 0.97656
Finished!
Testing Accuracy: 0.9921875
# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
#outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.132512, Training Accuracy= 0.30469
Iter 2560, Minibatch Loss= 2.001817, Training Accuracy= 0.28125
Iter 3840, Minibatch Loss= 1.523742, Training Accuracy= 0.51562
Iter 5120, Minibatch Loss= 1.272301, Training Accuracy= 0.64844
Iter 6400, Minibatch Loss= 1.060405, Training Accuracy= 0.65625
Iter 7680, Minibatch Loss= 0.889486, Training Accuracy= 0.69531
Iter 8960, Minibatch Loss= 0.854698, Training Accuracy= 0.69531
Iter 10240, Minibatch Loss= 0.786288, Training Accuracy= 0.77344
Iter 11520, Minibatch Loss= 0.758092, Training Accuracy= 0.77344
Iter 12800, Minibatch Loss= 0.784820, Training Accuracy= 0.77344
Iter 14080, Minibatch Loss= 0.383889, Training Accuracy= 0.91406
Iter 15360, Minibatch Loss= 0.458686, Training Accuracy= 0.82031
Iter 16640, Minibatch Loss= 0.446535, Training Accuracy= 0.85156
Iter 17920, Minibatch Loss= 0.593214, Training Accuracy= 0.83594
Iter 19200, Minibatch Loss= 0.483660, Training Accuracy= 0.85156
Iter 20480, Minibatch Loss= 0.413878, Training Accuracy= 0.87500
Iter 21760, Minibatch Loss= 0.438487, Training Accuracy= 0.85938
Iter 23040, Minibatch Loss= 0.440690, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.395759, Training Accuracy= 0.88281
Iter 25600, Minibatch Loss= 0.303816, Training Accuracy= 0.91406
Iter 26880, Minibatch Loss= 0.457287, Training Accuracy= 0.85156
Iter 28160, Minibatch Loss= 0.435749, Training Accuracy= 0.89062
Iter 29440, Minibatch Loss= 0.282319, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.341908, Training Accuracy= 0.90625
Iter 32000, Minibatch Loss= 0.413945, Training Accuracy= 0.84375
Iter 33280, Minibatch Loss= 0.277253, Training Accuracy= 0.90625
Iter 34560, Minibatch Loss= 0.265192, Training Accuracy= 0.92188
Iter 35840, Minibatch Loss= 0.273202, Training Accuracy= 0.89844
Iter 37120, Minibatch Loss= 0.188953, Training Accuracy= 0.95312
Iter 38400, Minibatch Loss= 0.401897, Training Accuracy= 0.89844
Iter 39680, Minibatch Loss= 0.277849, Training Accuracy= 0.91406
Iter 40960, Minibatch Loss= 0.256338, Training Accuracy= 0.91406
Iter 42240, Minibatch Loss= 0.259410, Training Accuracy= 0.92188
Iter 43520, Minibatch Loss= 0.286601, Training Accuracy= 0.95312
Iter 44800, Minibatch Loss= 0.163870, Training Accuracy= 0.96094
Iter 46080, Minibatch Loss= 0.161053, Training Accuracy= 0.96094
Iter 47360, Minibatch Loss= 0.250546, Training Accuracy= 0.92969
Iter 48640, Minibatch Loss= 0.134387, Training Accuracy= 0.96094
Iter 49920, Minibatch Loss= 0.173047, Training Accuracy= 0.92969
Iter 51200, Minibatch Loss= 0.337741, Training Accuracy= 0.90625
Iter 52480, Minibatch Loss= 0.207449, Training Accuracy= 0.93750
Iter 53760, Minibatch Loss= 0.156933, Training Accuracy= 0.94531
Iter 55040, Minibatch Loss= 0.199040, Training Accuracy= 0.94531
Iter 56320, Minibatch Loss= 0.147556, Training Accuracy= 0.95312
Iter 57600, Minibatch Loss= 0.148920, Training Accuracy= 0.94531
Iter 58880, Minibatch Loss= 0.144406, Training Accuracy= 0.94531
Iter 60160, Minibatch Loss= 0.114759, Training Accuracy= 0.96875
Iter 61440, Minibatch Loss= 0.145373, Training Accuracy= 0.95312
Iter 62720, Minibatch Loss= 0.154870, Training Accuracy= 0.95312
Iter 64000, Minibatch Loss= 0.176493, Training Accuracy= 0.95312
Iter 65280, Minibatch Loss= 0.124761, Training Accuracy= 0.96094
Iter 66560, Minibatch Loss= 0.060221, Training Accuracy= 0.98438
Iter 67840, Minibatch Loss= 0.154883, Training Accuracy= 0.95312
Iter 69120, Minibatch Loss= 0.141949, Training Accuracy= 0.95312
Iter 70400, Minibatch Loss= 0.289304, Training Accuracy= 0.91406
Iter 71680, Minibatch Loss= 0.167781, Training Accuracy= 0.94531
Iter 72960, Minibatch Loss= 0.239404, Training Accuracy= 0.93750
Iter 74240, Minibatch Loss= 0.138790, Training Accuracy= 0.96094
Iter 75520, Minibatch Loss= 0.176216, Training Accuracy= 0.92969
Iter 76800, Minibatch Loss= 0.179790, Training Accuracy= 0.95312
Iter 78080, Minibatch Loss= 0.157953, Training Accuracy= 0.93750
Iter 79360, Minibatch Loss= 0.213766, Training Accuracy= 0.92188
Iter 80640, Minibatch Loss= 0.080645, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.191703, Training Accuracy= 0.95312
Iter 83200, Minibatch Loss= 0.172994, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.210067, Training Accuracy= 0.93750
Iter 85760, Minibatch Loss= 0.165411, Training Accuracy= 0.94531
Iter 87040, Minibatch Loss= 0.087349, Training Accuracy= 0.97656
Iter 88320, Minibatch Loss= 0.189240, Training Accuracy= 0.93750
Iter 89600, Minibatch Loss= 0.153909, Training Accuracy= 0.94531
Iter 90880, Minibatch Loss= 0.231399, Training Accuracy= 0.92188
Iter 92160, Minibatch Loss= 0.132192, Training Accuracy= 0.95312
Iter 93440, Minibatch Loss= 0.078557, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.081286, Training Accuracy= 0.98438
Iter 96000, Minibatch Loss= 0.154948, Training Accuracy= 0.96094
Iter 97280, Minibatch Loss= 0.132285, Training Accuracy= 0.95312
Iter 98560, Minibatch Loss= 0.107341, Training Accuracy= 0.97656
Iter 99840, Minibatch Loss= 0.068848, Training Accuracy= 1.00000
Finished!
Testing Accuracy: 0.984375
实例58：构造单层GRU网络对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
gru = tf.contrib.rnn.GRUCell(n_hidden)
outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
#outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.110982, Training Accuracy= 0.35156
Iter 2560, Minibatch Loss= 1.813759, Training Accuracy= 0.36719
Iter 3840, Minibatch Loss= 1.430715, Training Accuracy= 0.57812
Iter 5120, Minibatch Loss= 1.243752, Training Accuracy= 0.53906
Iter 6400, Minibatch Loss= 1.036775, Training Accuracy= 0.66406
Iter 7680, Minibatch Loss= 0.958580, Training Accuracy= 0.65625
Iter 8960, Minibatch Loss= 0.797127, Training Accuracy= 0.75000
Iter 10240, Minibatch Loss= 0.834968, Training Accuracy= 0.71094
Iter 11520, Minibatch Loss= 0.762974, Training Accuracy= 0.72656
Iter 12800, Minibatch Loss= 0.702314, Training Accuracy= 0.78125
Iter 14080, Minibatch Loss= 0.631187, Training Accuracy= 0.75000
Iter 15360, Minibatch Loss= 0.601635, Training Accuracy= 0.82031
Iter 16640, Minibatch Loss= 0.539469, Training Accuracy= 0.84375
Iter 17920, Minibatch Loss= 0.446828, Training Accuracy= 0.89844
Iter 19200, Minibatch Loss= 0.491736, Training Accuracy= 0.86719
Iter 20480, Minibatch Loss= 0.443790, Training Accuracy= 0.85156
Iter 21760, Minibatch Loss= 0.360417, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.399639, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.365533, Training Accuracy= 0.90625
Iter 25600, Minibatch Loss= 0.361865, Training Accuracy= 0.89062
Iter 26880, Minibatch Loss= 0.314257, Training Accuracy= 0.89844
Iter 28160, Minibatch Loss= 0.323813, Training Accuracy= 0.88281
Iter 29440, Minibatch Loss= 0.258019, Training Accuracy= 0.92969
Iter 30720, Minibatch Loss= 0.362180, Training Accuracy= 0.90625
Iter 32000, Minibatch Loss= 0.300678, Training Accuracy= 0.90625
Iter 33280, Minibatch Loss= 0.345444, Training Accuracy= 0.88281
Iter 34560, Minibatch Loss= 0.315548, Training Accuracy= 0.90625
Iter 35840, Minibatch Loss= 0.149648, Training Accuracy= 0.94531
Iter 37120, Minibatch Loss= 0.208241, Training Accuracy= 0.95312
Iter 38400, Minibatch Loss= 0.240656, Training Accuracy= 0.92188
Iter 39680, Minibatch Loss= 0.242481, Training Accuracy= 0.94531
Iter 40960, Minibatch Loss= 0.198655, Training Accuracy= 0.94531
Iter 42240, Minibatch Loss= 0.169093, Training Accuracy= 0.96094
Iter 43520, Minibatch Loss= 0.144338, Training Accuracy= 0.96094
Iter 44800, Minibatch Loss= 0.120352, Training Accuracy= 0.96875
Iter 46080, Minibatch Loss= 0.260023, Training Accuracy= 0.92969
Iter 47360, Minibatch Loss= 0.259179, Training Accuracy= 0.92188
Iter 48640, Minibatch Loss= 0.169763, Training Accuracy= 0.95312
Iter 49920, Minibatch Loss= 0.295729, Training Accuracy= 0.87500
Iter 51200, Minibatch Loss= 0.262093, Training Accuracy= 0.94531
Iter 52480, Minibatch Loss= 0.160087, Training Accuracy= 0.94531
Iter 53760, Minibatch Loss= 0.216548, Training Accuracy= 0.92969
Iter 55040, Minibatch Loss= 0.197999, Training Accuracy= 0.92969
Iter 56320, Minibatch Loss= 0.209844, Training Accuracy= 0.93750
Iter 57600, Minibatch Loss= 0.145806, Training Accuracy= 0.96094
Iter 58880, Minibatch Loss= 0.145609, Training Accuracy= 0.92969
Iter 60160, Minibatch Loss= 0.141855, Training Accuracy= 0.96875
Iter 61440, Minibatch Loss= 0.149774, Training Accuracy= 0.96094
Iter 62720, Minibatch Loss= 0.151638, Training Accuracy= 0.95312
Iter 64000, Minibatch Loss= 0.183334, Training Accuracy= 0.95312
Iter 65280, Minibatch Loss= 0.135904, Training Accuracy= 0.96094
Iter 66560, Minibatch Loss= 0.099675, Training Accuracy= 0.97656
Iter 67840, Minibatch Loss= 0.122692, Training Accuracy= 0.96094
Iter 69120, Minibatch Loss= 0.146226, Training Accuracy= 0.94531
Iter 70400, Minibatch Loss= 0.180213, Training Accuracy= 0.96094
Iter 71680, Minibatch Loss= 0.095388, Training Accuracy= 0.96875
Iter 72960, Minibatch Loss= 0.151607, Training Accuracy= 0.95312
Iter 74240, Minibatch Loss= 0.138024, Training Accuracy= 0.95312
Iter 75520, Minibatch Loss= 0.077417, Training Accuracy= 0.98438
Iter 76800, Minibatch Loss= 0.142044, Training Accuracy= 0.96094
Iter 78080, Minibatch Loss= 0.096436, Training Accuracy= 0.96875
Iter 79360, Minibatch Loss= 0.170768, Training Accuracy= 0.94531
Iter 80640, Minibatch Loss= 0.244333, Training Accuracy= 0.94531
Iter 81920, Minibatch Loss= 0.049825, Training Accuracy= 0.98438
Iter 83200, Minibatch Loss= 0.100651, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.100706, Training Accuracy= 0.96875
Iter 85760, Minibatch Loss= 0.125473, Training Accuracy= 0.96094
Iter 87040, Minibatch Loss= 0.049680, Training Accuracy= 0.99219
Iter 88320, Minibatch Loss= 0.077539, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.124252, Training Accuracy= 0.96094
Iter 90880, Minibatch Loss= 0.039592, Training Accuracy= 0.99219
Iter 92160, Minibatch Loss= 0.108718, Training Accuracy= 0.96875
Iter 93440, Minibatch Loss= 0.085434, Training Accuracy= 0.97656
Iter 94720, Minibatch Loss= 0.164223, Training Accuracy= 0.95312
Iter 96000, Minibatch Loss= 0.174773, Training Accuracy= 0.92969
Iter 97280, Minibatch Loss= 0.062409, Training Accuracy= 0.98438
Iter 98560, Minibatch Loss= 0.065892, Training Accuracy= 0.98438
Iter 99840, Minibatch Loss= 0.109494, Training Accuracy= 0.96875
Finished!
Testing Accuracy: 0.96875
实例59：创建动态单层RNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.117727, Training Accuracy= 0.28906
Iter 2560, Minibatch Loss= 1.873549, Training Accuracy= 0.37500
Iter 3840, Minibatch Loss= 1.643477, Training Accuracy= 0.46875
Iter 5120, Minibatch Loss= 1.384699, Training Accuracy= 0.54688
Iter 6400, Minibatch Loss= 1.180172, Training Accuracy= 0.60938
Iter 7680, Minibatch Loss= 1.117720, Training Accuracy= 0.59375
Iter 8960, Minibatch Loss= 0.758485, Training Accuracy= 0.72656
Iter 10240, Minibatch Loss= 0.766036, Training Accuracy= 0.76562
Iter 11520, Minibatch Loss= 0.644146, Training Accuracy= 0.80469
Iter 12800, Minibatch Loss= 0.611225, Training Accuracy= 0.82812
Iter 14080, Minibatch Loss= 0.442834, Training Accuracy= 0.84375
Iter 15360, Minibatch Loss= 0.449818, Training Accuracy= 0.88281
Iter 16640, Minibatch Loss= 0.438875, Training Accuracy= 0.87500
Iter 17920, Minibatch Loss= 0.267779, Training Accuracy= 0.91406
Iter 19200, Minibatch Loss= 0.243440, Training Accuracy= 0.95312
Iter 20480, Minibatch Loss= 0.444132, Training Accuracy= 0.85938
Iter 21760, Minibatch Loss= 0.346954, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.175107, Training Accuracy= 0.95312
Iter 24320, Minibatch Loss= 0.268770, Training Accuracy= 0.92969
Iter 25600, Minibatch Loss= 0.297496, Training Accuracy= 0.91406
Iter 26880, Minibatch Loss= 0.274186, Training Accuracy= 0.89844
Iter 28160, Minibatch Loss= 0.153380, Training Accuracy= 0.94531
Iter 29440, Minibatch Loss= 0.223070, Training Accuracy= 0.93750
Iter 30720, Minibatch Loss= 0.187295, Training Accuracy= 0.92188
Iter 32000, Minibatch Loss= 0.192284, Training Accuracy= 0.94531
Iter 33280, Minibatch Loss= 0.206862, Training Accuracy= 0.93750
Iter 34560, Minibatch Loss= 0.157154, Training Accuracy= 0.94531
Iter 35840, Minibatch Loss= 0.221760, Training Accuracy= 0.92188
Iter 37120, Minibatch Loss= 0.204846, Training Accuracy= 0.94531
Iter 38400, Minibatch Loss= 0.143488, Training Accuracy= 0.96094
Iter 39680, Minibatch Loss= 0.147712, Training Accuracy= 0.95312
Iter 40960, Minibatch Loss= 0.260166, Training Accuracy= 0.92188
Iter 42240, Minibatch Loss= 0.236482, Training Accuracy= 0.92188
Iter 43520, Minibatch Loss= 0.222228, Training Accuracy= 0.93750
Iter 44800, Minibatch Loss= 0.135309, Training Accuracy= 0.96094
Iter 46080, Minibatch Loss= 0.148444, Training Accuracy= 0.96094
Iter 47360, Minibatch Loss= 0.284828, Training Accuracy= 0.92188
Iter 48640, Minibatch Loss= 0.276086, Training Accuracy= 0.92188
Iter 49920, Minibatch Loss= 0.111534, Training Accuracy= 0.95312
Iter 51200, Minibatch Loss= 0.111628, Training Accuracy= 0.96094
Iter 52480, Minibatch Loss= 0.131188, Training Accuracy= 0.96875
Iter 53760, Minibatch Loss= 0.080626, Training Accuracy= 0.98438
Iter 55040, Minibatch Loss= 0.118804, Training Accuracy= 0.96094
Iter 56320, Minibatch Loss= 0.211226, Training Accuracy= 0.92969
Iter 57600, Minibatch Loss= 0.165107, Training Accuracy= 0.95312
Iter 58880, Minibatch Loss= 0.134429, Training Accuracy= 0.96094
Iter 60160, Minibatch Loss= 0.168475, Training Accuracy= 0.95312
Iter 61440, Minibatch Loss= 0.128811, Training Accuracy= 0.96094
Iter 62720, Minibatch Loss= 0.077244, Training Accuracy= 0.96094
Iter 64000, Minibatch Loss= 0.090807, Training Accuracy= 0.96875
Iter 65280, Minibatch Loss= 0.192585, Training Accuracy= 0.92188
Iter 66560, Minibatch Loss= 0.197945, Training Accuracy= 0.95312
Iter 67840, Minibatch Loss= 0.146314, Training Accuracy= 0.96875
Iter 69120, Minibatch Loss= 0.117953, Training Accuracy= 0.96875
Iter 70400, Minibatch Loss= 0.174820, Training Accuracy= 0.96875
Iter 71680, Minibatch Loss= 0.078789, Training Accuracy= 0.98438
Iter 72960, Minibatch Loss= 0.180256, Training Accuracy= 0.92188
Iter 74240, Minibatch Loss= 0.142464, Training Accuracy= 0.94531
Iter 75520, Minibatch Loss= 0.112260, Training Accuracy= 0.96875
Iter 76800, Minibatch Loss= 0.133794, Training Accuracy= 0.95312
Iter 78080, Minibatch Loss= 0.067401, Training Accuracy= 0.99219
Iter 79360, Minibatch Loss= 0.154360, Training Accuracy= 0.96094
Iter 80640, Minibatch Loss= 0.067086, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.110198, Training Accuracy= 0.97656
Iter 83200, Minibatch Loss= 0.149606, Training Accuracy= 0.95312
Iter 84480, Minibatch Loss= 0.119215, Training Accuracy= 0.96875
Iter 85760, Minibatch Loss= 0.051235, Training Accuracy= 0.98438
Iter 87040, Minibatch Loss= 0.094870, Training Accuracy= 0.96875
Iter 88320, Minibatch Loss= 0.073898, Training Accuracy= 0.98438
Iter 89600, Minibatch Loss= 0.087323, Training Accuracy= 0.96094
Iter 90880, Minibatch Loss= 0.088403, Training Accuracy= 0.96094
Iter 92160, Minibatch Loss= 0.120743, Training Accuracy= 0.97656
Iter 93440, Minibatch Loss= 0.061005, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.095413, Training Accuracy= 0.97656
Iter 96000, Minibatch Loss= 0.078382, Training Accuracy= 0.97656
Iter 97280, Minibatch Loss= 0.100441, Training Accuracy= 0.97656
Iter 98560, Minibatch Loss= 0.022794, Training Accuracy= 1.00000
Iter 99840, Minibatch Loss= 0.085918, Training Accuracy= 0.97656
Finished!
Testing Accuracy: 0.984375
实例60：静态多层LSTM对MNIST数据集分类
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
stacked_rnn = []
for i in range(3):
stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.892138, Training Accuracy= 0.27344
Iter 2560, Minibatch Loss= 1.520981, Training Accuracy= 0.42969
Iter 3840, Minibatch Loss= 1.271327, Training Accuracy= 0.53906
Iter 5120, Minibatch Loss= 0.980068, Training Accuracy= 0.65625
Iter 6400, Minibatch Loss= 0.812825, Training Accuracy= 0.74219
Iter 7680, Minibatch Loss= 0.807655, Training Accuracy= 0.74219
Iter 8960, Minibatch Loss= 0.761637, Training Accuracy= 0.77344
Iter 10240, Minibatch Loss= 0.580690, Training Accuracy= 0.82812
Iter 11520, Minibatch Loss= 0.802299, Training Accuracy= 0.72656
Iter 12800, Minibatch Loss= 0.557632, Training Accuracy= 0.82031
Iter 14080, Minibatch Loss= 0.580068, Training Accuracy= 0.85938
Iter 15360, Minibatch Loss= 0.365136, Training Accuracy= 0.88281
Iter 16640, Minibatch Loss= 0.554880, Training Accuracy= 0.83594
Iter 17920, Minibatch Loss= 0.392231, Training Accuracy= 0.88281
Iter 19200, Minibatch Loss= 0.239315, Training Accuracy= 0.92969
Iter 20480, Minibatch Loss= 0.266329, Training Accuracy= 0.92969
Iter 21760, Minibatch Loss= 0.239885, Training Accuracy= 0.93750
Iter 23040, Minibatch Loss= 0.308746, Training Accuracy= 0.89062
Iter 24320, Minibatch Loss= 0.405082, Training Accuracy= 0.86719
Iter 25600, Minibatch Loss= 0.306493, Training Accuracy= 0.92969
Iter 26880, Minibatch Loss= 0.297999, Training Accuracy= 0.92188
Iter 28160, Minibatch Loss= 0.205317, Training Accuracy= 0.94531
Iter 29440, Minibatch Loss= 0.209697, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.224113, Training Accuracy= 0.92188
Iter 32000, Minibatch Loss= 0.202094, Training Accuracy= 0.93750
Iter 33280, Minibatch Loss= 0.209670, Training Accuracy= 0.92188
Iter 34560, Minibatch Loss= 0.275021, Training Accuracy= 0.92188
Iter 35840, Minibatch Loss= 0.141351, Training Accuracy= 0.95312
Iter 37120, Minibatch Loss= 0.172745, Training Accuracy= 0.92188
Iter 38400, Minibatch Loss= 0.228708, Training Accuracy= 0.92969
Iter 39680, Minibatch Loss= 0.245980, Training Accuracy= 0.92188
Iter 40960, Minibatch Loss= 0.216683, Training Accuracy= 0.92188
Iter 42240, Minibatch Loss= 0.183437, Training Accuracy= 0.96094
Iter 43520, Minibatch Loss= 0.249385, Training Accuracy= 0.92969
Iter 44800, Minibatch Loss= 0.275260, Training Accuracy= 0.91406
Iter 46080, Minibatch Loss= 0.174156, Training Accuracy= 0.92969
Iter 47360, Minibatch Loss= 0.261219, Training Accuracy= 0.90625
Iter 48640, Minibatch Loss= 0.187510, Training Accuracy= 0.93750
Iter 49920, Minibatch Loss= 0.090633, Training Accuracy= 0.98438
Iter 51200, Minibatch Loss= 0.061939, Training Accuracy= 0.97656
Iter 52480, Minibatch Loss= 0.075695, Training Accuracy= 0.97656
Iter 53760, Minibatch Loss= 0.322783, Training Accuracy= 0.92188
Iter 55040, Minibatch Loss= 0.126282, Training Accuracy= 0.96875
Iter 56320, Minibatch Loss= 0.119054, Training Accuracy= 0.96875
Iter 57600, Minibatch Loss= 0.213464, Training Accuracy= 0.94531
Iter 58880, Minibatch Loss= 0.083522, Training Accuracy= 0.96094
Iter 60160, Minibatch Loss= 0.094831, Training Accuracy= 0.98438
Iter 61440, Minibatch Loss= 0.096521, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.173445, Training Accuracy= 0.93750
Iter 64000, Minibatch Loss= 0.119984, Training Accuracy= 0.96875
Iter 65280, Minibatch Loss= 0.180508, Training Accuracy= 0.95312
Iter 66560, Minibatch Loss= 0.068011, Training Accuracy= 0.98438
Iter 67840, Minibatch Loss= 0.210539, Training Accuracy= 0.95312
Iter 69120, Minibatch Loss= 0.097277, Training Accuracy= 0.96875
Iter 70400, Minibatch Loss= 0.196134, Training Accuracy= 0.95312
Iter 71680, Minibatch Loss= 0.091252, Training Accuracy= 0.96094
Iter 72960, Minibatch Loss= 0.144938, Training Accuracy= 0.95312
Iter 74240, Minibatch Loss= 0.101681, Training Accuracy= 0.96875
Iter 75520, Minibatch Loss= 0.057837, Training Accuracy= 0.97656
Iter 76800, Minibatch Loss= 0.095161, Training Accuracy= 0.97656
Iter 78080, Minibatch Loss= 0.091807, Training Accuracy= 0.98438
Iter 79360, Minibatch Loss= 0.063520, Training Accuracy= 0.99219
Iter 80640, Minibatch Loss= 0.087506, Training Accuracy= 0.96875
Iter 81920, Minibatch Loss= 0.264015, Training Accuracy= 0.93750
Iter 83200, Minibatch Loss= 0.163464, Training Accuracy= 0.95312
Iter 84480, Minibatch Loss= 0.074320, Training Accuracy= 0.98438
Iter 85760, Minibatch Loss= 0.087142, Training Accuracy= 0.96875
Iter 87040, Minibatch Loss= 0.061768, Training Accuracy= 0.97656
Iter 88320, Minibatch Loss= 0.129232, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.054522, Training Accuracy= 0.98438
Iter 90880, Minibatch Loss= 0.060756, Training Accuracy= 0.97656
Iter 92160, Minibatch Loss= 0.034073, Training Accuracy= 1.00000
Iter 93440, Minibatch Loss= 0.112970, Training Accuracy= 0.97656
Iter 94720, Minibatch Loss= 0.088832, Training Accuracy= 0.96875
Iter 96000, Minibatch Loss= 0.093209, Training Accuracy= 0.96875
Iter 97280, Minibatch Loss= 0.098466, Training Accuracy= 0.97656
Iter 98560, Minibatch Loss= 0.085677, Training Accuracy= 0.97656
Iter 99840, Minibatch Loss= 0.102284, Training Accuracy= 0.96875
Finished!
Testing Accuracy: 1.0
实例61：静态多层RNN-LSTM连接GRU对MNIST数据集分类
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
#
#stacked_rnn = []
#for i in range(3):
# stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
gru = tf.contrib.rnn.GRUCell(n_hidden*2)
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden)
mcell = tf.contrib.rnn.MultiRNNCell([lstm_cell,gru])
x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.808671, Training Accuracy= 0.42188
Iter 2560, Minibatch Loss= 1.288418, Training Accuracy= 0.59375
Iter 3840, Minibatch Loss= 0.954410, Training Accuracy= 0.75781
Iter 5120, Minibatch Loss= 0.873409, Training Accuracy= 0.70312
Iter 6400, Minibatch Loss= 0.872713, Training Accuracy= 0.66406
Iter 7680, Minibatch Loss= 0.618704, Training Accuracy= 0.83594
Iter 8960, Minibatch Loss= 0.426509, Training Accuracy= 0.88281
Iter 10240, Minibatch Loss= 0.480236, Training Accuracy= 0.86719
Iter 11520, Minibatch Loss= 0.429108, Training Accuracy= 0.84375
Iter 12800, Minibatch Loss= 0.552932, Training Accuracy= 0.82812
Iter 14080, Minibatch Loss= 0.362248, Training Accuracy= 0.87500
Iter 15360, Minibatch Loss= 0.265852, Training Accuracy= 0.92969
Iter 16640, Minibatch Loss= 0.315052, Training Accuracy= 0.87500
Iter 17920, Minibatch Loss= 0.259191, Training Accuracy= 0.93750
Iter 19200, Minibatch Loss= 0.360970, Training Accuracy= 0.89844
Iter 20480, Minibatch Loss= 0.292491, Training Accuracy= 0.94531
Iter 21760, Minibatch Loss= 0.259818, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.293232, Training Accuracy= 0.93750
Iter 24320, Minibatch Loss= 0.244462, Training Accuracy= 0.94531
Iter 25600, Minibatch Loss= 0.227099, Training Accuracy= 0.93750
Iter 26880, Minibatch Loss= 0.231890, Training Accuracy= 0.92969
Iter 28160, Minibatch Loss= 0.196979, Training Accuracy= 0.94531
Iter 29440, Minibatch Loss= 0.165492, Training Accuracy= 0.95312
Iter 30720, Minibatch Loss= 0.252359, Training Accuracy= 0.95312
Iter 32000, Minibatch Loss= 0.237196, Training Accuracy= 0.94531
Iter 33280, Minibatch Loss= 0.197849, Training Accuracy= 0.95312
Iter 34560, Minibatch Loss= 0.126519, Training Accuracy= 0.96094
Iter 35840, Minibatch Loss= 0.173198, Training Accuracy= 0.95312
Iter 37120, Minibatch Loss= 0.176549, Training Accuracy= 0.95312
Iter 38400, Minibatch Loss= 0.206727, Training Accuracy= 0.95312
Iter 39680, Minibatch Loss= 0.170020, Training Accuracy= 0.94531
Iter 40960, Minibatch Loss= 0.211021, Training Accuracy= 0.91406
Iter 42240, Minibatch Loss= 0.174285, Training Accuracy= 0.93750
Iter 43520, Minibatch Loss= 0.074847, Training Accuracy= 0.99219
Iter 44800, Minibatch Loss= 0.188610, Training Accuracy= 0.96875
Iter 46080, Minibatch Loss= 0.098489, Training Accuracy= 0.96875
Iter 47360, Minibatch Loss= 0.164228, Training Accuracy= 0.96094
Iter 48640, Minibatch Loss= 0.099976, Training Accuracy= 0.97656
Iter 49920, Minibatch Loss= 0.185240, Training Accuracy= 0.93750
Iter 51200, Minibatch Loss= 0.146018, Training Accuracy= 0.95312
Iter 52480, Minibatch Loss= 0.085702, Training Accuracy= 0.97656
Iter 53760, Minibatch Loss= 0.079165, Training Accuracy= 0.98438
Iter 55040, Minibatch Loss= 0.230488, Training Accuracy= 0.94531
Iter 56320, Minibatch Loss= 0.026857, Training Accuracy= 1.00000
Iter 57600, Minibatch Loss= 0.189286, Training Accuracy= 0.93750
Iter 58880, Minibatch Loss= 0.085770, Training Accuracy= 0.97656
Iter 60160, Minibatch Loss= 0.102213, Training Accuracy= 0.96094
Iter 61440, Minibatch Loss= 0.087951, Training Accuracy= 0.97656
Iter 62720, Minibatch Loss= 0.105146, Training Accuracy= 0.96094
Iter 64000, Minibatch Loss= 0.150521, Training Accuracy= 0.96094
Iter 65280, Minibatch Loss= 0.051553, Training Accuracy= 0.99219
Iter 66560, Minibatch Loss= 0.044942, Training Accuracy= 0.98438
Iter 67840, Minibatch Loss= 0.093745, Training Accuracy= 0.95312
Iter 69120, Minibatch Loss= 0.091557, Training Accuracy= 0.96094
Iter 70400, Minibatch Loss= 0.079020, Training Accuracy= 0.97656
Iter 71680, Minibatch Loss= 0.107035, Training Accuracy= 0.96875
Iter 72960, Minibatch Loss= 0.103098, Training Accuracy= 0.96094
Iter 74240, Minibatch Loss= 0.137766, Training Accuracy= 0.96875
Iter 75520, Minibatch Loss= 0.087706, Training Accuracy= 0.96875
Iter 76800, Minibatch Loss= 0.056394, Training Accuracy= 0.99219
Iter 78080, Minibatch Loss= 0.097080, Training Accuracy= 0.97656
Iter 79360, Minibatch Loss= 0.064108, Training Accuracy= 0.97656
Iter 80640, Minibatch Loss= 0.081608, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.074628, Training Accuracy= 0.98438
Iter 83200, Minibatch Loss= 0.173194, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.166146, Training Accuracy= 0.94531
Iter 85760, Minibatch Loss= 0.132719, Training Accuracy= 0.97656
Iter 87040, Minibatch Loss= 0.119572, Training Accuracy= 0.96875
Iter 88320, Minibatch Loss= 0.030019, Training Accuracy= 1.00000
Iter 89600, Minibatch Loss= 0.176066, Training Accuracy= 0.96875
Iter 90880, Minibatch Loss= 0.069413, Training Accuracy= 0.96875
Iter 92160, Minibatch Loss= 0.112738, Training Accuracy= 0.97656
Iter 93440, Minibatch Loss= 0.069958, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.037432, Training Accuracy= 0.99219
Iter 96000, Minibatch Loss= 0.082820, Training Accuracy= 0.98438
Iter 97280, Minibatch Loss= 0.121944, Training Accuracy= 0.96875
Iter 98560, Minibatch Loss= 0.058363, Training Accuracy= 0.99219
Iter 99840, Minibatch Loss= 0.139464, Training Accuracy= 0.96875
Finished!
Testing Accuracy: 0.97
实例62：动态多层RNN对MNIST分类
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
#
#stacked_rnn = []
#for i in range(3):
# stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
gru = tf.contrib.rnn.GRUCell(n_hidden*2)
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden)
mcell = tf.contrib.rnn.MultiRNNCell([lstm_cell,gru])
#x1 = tf.unstack(x, n_steps, 1)
#outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
#pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
outputs,states = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序，取最后一个时序outputs[-1]=(?,256)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.786891, Training Accuracy= 0.36719
Iter 2560, Minibatch Loss= 1.360941, Training Accuracy= 0.55469
Iter 3840, Minibatch Loss= 1.208105, Training Accuracy= 0.60156
Iter 5120, Minibatch Loss= 1.075533, Training Accuracy= 0.62500
Iter 6400, Minibatch Loss= 0.780158, Training Accuracy= 0.72656
Iter 7680, Minibatch Loss= 0.785984, Training Accuracy= 0.71875
Iter 8960, Minibatch Loss= 0.459019, Training Accuracy= 0.87500
Iter 10240, Minibatch Loss= 0.699168, Training Accuracy= 0.78125
Iter 11520, Minibatch Loss= 0.549069, Training Accuracy= 0.82031
Iter 12800, Minibatch Loss= 0.412937, Training Accuracy= 0.83594
Iter 14080, Minibatch Loss= 0.359638, Training Accuracy= 0.88281
Iter 15360, Minibatch Loss= 0.358867, Training Accuracy= 0.89844
Iter 16640, Minibatch Loss= 0.384718, Training Accuracy= 0.85938
Iter 17920, Minibatch Loss= 0.344854, Training Accuracy= 0.90625
Iter 19200, Minibatch Loss= 0.320385, Training Accuracy= 0.89062
Iter 20480, Minibatch Loss= 0.424948, Training Accuracy= 0.89062
Iter 21760, Minibatch Loss= 0.237800, Training Accuracy= 0.92969
Iter 23040, Minibatch Loss= 0.153665, Training Accuracy= 0.96094
Iter 24320, Minibatch Loss= 0.269827, Training Accuracy= 0.92188
Iter 25600, Minibatch Loss= 0.209402, Training Accuracy= 0.94531
Iter 26880, Minibatch Loss= 0.191818, Training Accuracy= 0.96094
Iter 28160, Minibatch Loss= 0.222970, Training Accuracy= 0.91406
Iter 29440, Minibatch Loss= 0.114014, Training Accuracy= 0.98438
Iter 30720, Minibatch Loss= 0.237388, Training Accuracy= 0.92969
Iter 32000, Minibatch Loss= 0.107080, Training Accuracy= 0.97656
Iter 33280, Minibatch Loss= 0.160952, Training Accuracy= 0.94531
Iter 34560, Minibatch Loss= 0.153807, Training Accuracy= 0.95312
Iter 35840, Minibatch Loss= 0.162314, Training Accuracy= 0.93750
Iter 37120, Minibatch Loss= 0.158119, Training Accuracy= 0.94531
Iter 38400, Minibatch Loss= 0.075459, Training Accuracy= 0.97656
Iter 39680, Minibatch Loss= 0.145804, Training Accuracy= 0.96875
Iter 40960, Minibatch Loss= 0.099279, Training Accuracy= 0.99219
Iter 42240, Minibatch Loss= 0.171049, Training Accuracy= 0.94531
Iter 43520, Minibatch Loss= 0.231703, Training Accuracy= 0.93750
Iter 44800, Minibatch Loss= 0.102913, Training Accuracy= 0.97656
Iter 46080, Minibatch Loss= 0.091480, Training Accuracy= 0.98438
Iter 47360, Minibatch Loss= 0.101064, Training Accuracy= 0.96875
Iter 48640, Minibatch Loss= 0.187488, Training Accuracy= 0.94531
Iter 49920, Minibatch Loss= 0.115087, Training Accuracy= 0.96094
Iter 51200, Minibatch Loss= 0.059695, Training Accuracy= 0.99219
Iter 52480, Minibatch Loss= 0.112815, Training Accuracy= 0.95312
Iter 53760, Minibatch Loss= 0.177814, Training Accuracy= 0.94531
Iter 55040, Minibatch Loss= 0.082340, Training Accuracy= 0.98438
Iter 56320, Minibatch Loss= 0.110558, Training Accuracy= 0.96875
Iter 57600, Minibatch Loss= 0.103817, Training Accuracy= 0.97656
Iter 58880, Minibatch Loss= 0.084542, Training Accuracy= 0.97656
Iter 60160, Minibatch Loss= 0.173816, Training Accuracy= 0.95312
Iter 61440, Minibatch Loss= 0.084009, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.103503, Training Accuracy= 0.96875
Iter 64000, Minibatch Loss= 0.054017, Training Accuracy= 0.97656
Iter 65280, Minibatch Loss= 0.040551, Training Accuracy= 0.99219
Iter 66560, Minibatch Loss= 0.156163, Training Accuracy= 0.95312
Iter 67840, Minibatch Loss= 0.082662, Training Accuracy= 0.96875
Iter 69120, Minibatch Loss= 0.038192, Training Accuracy= 1.00000
Iter 70400, Minibatch Loss= 0.167307, Training Accuracy= 0.96094
Iter 71680, Minibatch Loss= 0.112745, Training Accuracy= 0.97656
Iter 72960, Minibatch Loss= 0.053178, Training Accuracy= 0.98438
Iter 74240, Minibatch Loss= 0.150620, Training Accuracy= 0.95312
Iter 75520, Minibatch Loss= 0.086217, Training Accuracy= 0.98438
Iter 76800, Minibatch Loss= 0.108635, Training Accuracy= 0.98438
Iter 78080, Minibatch Loss= 0.078446, Training Accuracy= 0.98438
Iter 79360, Minibatch Loss= 0.117279, Training Accuracy= 0.96875
Iter 80640, Minibatch Loss= 0.046549, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.082791, Training Accuracy= 0.97656
Iter 83200, Minibatch Loss= 0.047810, Training Accuracy= 0.98438
Iter 84480, Minibatch Loss= 0.048099, Training Accuracy= 0.99219
Iter 85760, Minibatch Loss= 0.109521, Training Accuracy= 0.96094
Iter 87040, Minibatch Loss= 0.054557, Training Accuracy= 0.98438
Iter 88320, Minibatch Loss= 0.035168, Training Accuracy= 0.99219
Iter 89600, Minibatch Loss= 0.077837, Training Accuracy= 0.98438
Iter 90880, Minibatch Loss= 0.162258, Training Accuracy= 0.94531
Iter 92160, Minibatch Loss= 0.121832, Training Accuracy= 0.97656
Iter 93440, Minibatch Loss= 0.040085, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.047824, Training Accuracy= 0.98438
Iter 96000, Minibatch Loss= 0.112367, Training Accuracy= 0.96094
Iter 97280, Minibatch Loss= 0.077551, Training Accuracy= 0.99219
Iter 98560, Minibatch Loss= 0.093468, Training Accuracy= 0.95312
Iter 99840, Minibatch Loss= 0.088767, Training Accuracy= 0.98438
Finished!
Testing Accuracy: 0.99
实例63：构建单层动态BiRNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# 反向cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,lstm_bw_cell,x,
dtype=tf.float32)
print(len(outputs),outputs[0].shape,outputs[1].shape)
outputs = tf.concat(outputs, 2)
outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.160943, Training Accuracy= 0.27344
Iter 2560, Minibatch Loss= 1.997462, Training Accuracy= 0.33594
Iter 3840, Minibatch Loss= 1.570268, Training Accuracy= 0.42188
Iter 5120, Minibatch Loss= 1.373489, Training Accuracy= 0.48438
Iter 6400, Minibatch Loss= 1.088359, Training Accuracy= 0.65625
Iter 7680, Minibatch Loss= 0.993419, Training Accuracy= 0.70312
Iter 8960, Minibatch Loss= 0.906906, Training Accuracy= 0.72656
Iter 10240, Minibatch Loss= 0.781511, Training Accuracy= 0.73438
Iter 11520, Minibatch Loss= 0.745537, Training Accuracy= 0.74219
Iter 12800, Minibatch Loss= 0.687559, Training Accuracy= 0.79688
Iter 14080, Minibatch Loss= 0.536104, Training Accuracy= 0.83594
Iter 15360, Minibatch Loss= 0.499159, Training Accuracy= 0.85938
Iter 16640, Minibatch Loss= 0.525461, Training Accuracy= 0.84375
Iter 17920, Minibatch Loss= 0.458517, Training Accuracy= 0.87500
Iter 19200, Minibatch Loss= 0.525781, Training Accuracy= 0.82031
Iter 20480, Minibatch Loss= 0.398223, Training Accuracy= 0.89062
Iter 21760, Minibatch Loss= 0.343273, Training Accuracy= 0.89062
Iter 23040, Minibatch Loss= 0.294032, Training Accuracy= 0.90625
Iter 24320, Minibatch Loss= 0.333711, Training Accuracy= 0.91406
Iter 25600, Minibatch Loss= 0.303608, Training Accuracy= 0.89062
Iter 26880, Minibatch Loss= 0.356443, Training Accuracy= 0.89844
Iter 28160, Minibatch Loss= 0.318978, Training Accuracy= 0.90625
Iter 29440, Minibatch Loss= 0.346513, Training Accuracy= 0.91406
Iter 30720, Minibatch Loss= 0.351226, Training Accuracy= 0.89062
Iter 32000, Minibatch Loss= 0.262708, Training Accuracy= 0.90625
Iter 33280, Minibatch Loss= 0.250729, Training Accuracy= 0.91406
Iter 34560, Minibatch Loss= 0.429823, Training Accuracy= 0.85156
Iter 35840, Minibatch Loss= 0.348004, Training Accuracy= 0.91406
Iter 37120, Minibatch Loss= 0.325374, Training Accuracy= 0.91406
Iter 38400, Minibatch Loss= 0.247374, Training Accuracy= 0.89062
Iter 39680, Minibatch Loss= 0.402448, Training Accuracy= 0.87500
Iter 40960, Minibatch Loss= 0.314637, Training Accuracy= 0.92188
Iter 42240, Minibatch Loss= 0.252825, Training Accuracy= 0.92969
Iter 43520, Minibatch Loss= 0.280104, Training Accuracy= 0.91406
Iter 44800, Minibatch Loss= 0.251025, Training Accuracy= 0.92969
Iter 46080, Minibatch Loss= 0.336563, Training Accuracy= 0.89062
Iter 47360, Minibatch Loss= 0.195152, Training Accuracy= 0.94531
Iter 48640, Minibatch Loss= 0.202264, Training Accuracy= 0.94531
Iter 49920, Minibatch Loss= 0.203700, Training Accuracy= 0.92188
Iter 51200, Minibatch Loss= 0.207661, Training Accuracy= 0.96094
Iter 52480, Minibatch Loss= 0.264310, Training Accuracy= 0.92969
Iter 53760, Minibatch Loss= 0.190808, Training Accuracy= 0.94531
Iter 55040, Minibatch Loss= 0.083677, Training Accuracy= 0.97656
Iter 56320, Minibatch Loss= 0.236587, Training Accuracy= 0.92188
Iter 57600, Minibatch Loss= 0.242121, Training Accuracy= 0.92969
Iter 58880, Minibatch Loss= 0.182118, Training Accuracy= 0.96094
Iter 60160, Minibatch Loss= 0.174736, Training Accuracy= 0.94531
Iter 61440, Minibatch Loss= 0.178520, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.163360, Training Accuracy= 0.93750
Iter 64000, Minibatch Loss= 0.219186, Training Accuracy= 0.92969
Iter 65280, Minibatch Loss= 0.082264, Training Accuracy= 0.98438
Iter 66560, Minibatch Loss= 0.171661, Training Accuracy= 0.95312
Iter 67840, Minibatch Loss= 0.210177, Training Accuracy= 0.96094
Iter 69120, Minibatch Loss= 0.301095, Training Accuracy= 0.92969
Iter 70400, Minibatch Loss= 0.082820, Training Accuracy= 0.97656
Iter 71680, Minibatch Loss= 0.222693, Training Accuracy= 0.93750
Iter 72960, Minibatch Loss= 0.164202, Training Accuracy= 0.92969
Iter 74240, Minibatch Loss= 0.158589, Training Accuracy= 0.95312
Iter 75520, Minibatch Loss= 0.155147, Training Accuracy= 0.96094
Iter 76800, Minibatch Loss= 0.139993, Training Accuracy= 0.95312
Iter 78080, Minibatch Loss= 0.138517, Training Accuracy= 0.95312
Iter 79360, Minibatch Loss= 0.184054, Training Accuracy= 0.94531
Iter 80640, Minibatch Loss= 0.207642, Training Accuracy= 0.92188
Iter 81920, Minibatch Loss= 0.127576, Training Accuracy= 0.96094
Iter 83200, Minibatch Loss= 0.157001, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.169914, Training Accuracy= 0.94531
Iter 85760, Minibatch Loss= 0.133477, Training Accuracy= 0.96094
Iter 87040, Minibatch Loss= 0.244336, Training Accuracy= 0.89844
Iter 88320, Minibatch Loss= 0.071679, Training Accuracy= 0.99219
Iter 89600, Minibatch Loss= 0.123343, Training Accuracy= 0.96094
Iter 90880, Minibatch Loss= 0.217972, Training Accuracy= 0.92188
Iter 92160, Minibatch Loss= 0.082326, Training Accuracy= 0.99219
Iter 93440, Minibatch Loss= 0.206144, Training Accuracy= 0.94531
Iter 94720, Minibatch Loss= 0.119195, Training Accuracy= 0.97656
Iter 96000, Minibatch Loss= 0.076865, Training Accuracy= 0.97656
Iter 97280, Minibatch Loss= 0.208217, Training Accuracy= 0.92969
Iter 98560, Minibatch Loss= 0.143691, Training Accuracy= 0.94531
Iter 99840, Minibatch Loss= 0.119417, Training Accuracy= 0.97656
Finished!
Testing Accuracy: 0.9765625
实例64：构建单层静态BiRNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# 反向cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
dtype=tf.float32)
print(outputs[0].shape,len(outputs))
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.198292, Training Accuracy= 0.34375
Iter 2560, Minibatch Loss= 1.922280, Training Accuracy= 0.33594
Iter 3840, Minibatch Loss= 1.619660, Training Accuracy= 0.48438
Iter 5120, Minibatch Loss= 1.287644, Training Accuracy= 0.57812
Iter 6400, Minibatch Loss= 1.054544, Training Accuracy= 0.63281
Iter 7680, Minibatch Loss= 0.970391, Training Accuracy= 0.64844
Iter 8960, Minibatch Loss= 0.785846, Training Accuracy= 0.75000
Iter 10240, Minibatch Loss= 0.735042, Training Accuracy= 0.77344
Iter 11520, Minibatch Loss= 0.709633, Training Accuracy= 0.78906
Iter 12800, Minibatch Loss= 0.494802, Training Accuracy= 0.85938
Iter 14080, Minibatch Loss= 0.587061, Training Accuracy= 0.82812
Iter 15360, Minibatch Loss= 0.571160, Training Accuracy= 0.80469
Iter 16640, Minibatch Loss= 0.576181, Training Accuracy= 0.80469
Iter 17920, Minibatch Loss= 0.425966, Training Accuracy= 0.89062
Iter 19200, Minibatch Loss= 0.334115, Training Accuracy= 0.91406
Iter 20480, Minibatch Loss= 0.367955, Training Accuracy= 0.89844
Iter 21760, Minibatch Loss= 0.314265, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.325072, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.427572, Training Accuracy= 0.86719
Iter 25600, Minibatch Loss= 0.407501, Training Accuracy= 0.86719
Iter 26880, Minibatch Loss= 0.279054, Training Accuracy= 0.92969
Iter 28160, Minibatch Loss= 0.250573, Training Accuracy= 0.91406
Iter 29440, Minibatch Loss= 0.243975, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.238282, Training Accuracy= 0.93750
Iter 32000, Minibatch Loss= 0.205076, Training Accuracy= 0.92188
Iter 33280, Minibatch Loss= 0.346559, Training Accuracy= 0.89844
Iter 34560, Minibatch Loss= 0.247907, Training Accuracy= 0.92188
Iter 35840, Minibatch Loss= 0.277912, Training Accuracy= 0.90625
Iter 37120, Minibatch Loss= 0.290173, Training Accuracy= 0.92188
Iter 38400, Minibatch Loss= 0.301018, Training Accuracy= 0.89062
Iter 39680, Minibatch Loss= 0.269850, Training Accuracy= 0.90625
Iter 40960, Minibatch Loss= 0.276790, Training Accuracy= 0.91406
Iter 42240, Minibatch Loss= 0.234881, Training Accuracy= 0.95312
Iter 43520, Minibatch Loss= 0.276319, Training Accuracy= 0.91406
Iter 44800, Minibatch Loss= 0.232567, Training Accuracy= 0.92188
Iter 46080, Minibatch Loss= 0.319510, Training Accuracy= 0.89062
Iter 47360, Minibatch Loss= 0.148028, Training Accuracy= 0.95312
Iter 48640, Minibatch Loss= 0.176917, Training Accuracy= 0.95312
Iter 49920, Minibatch Loss= 0.236975, Training Accuracy= 0.92969
Iter 51200, Minibatch Loss= 0.129818, Training Accuracy= 0.96094
Iter 52480, Minibatch Loss= 0.238299, Training Accuracy= 0.94531
Iter 53760, Minibatch Loss= 0.156861, Training Accuracy= 0.95312
Iter 55040, Minibatch Loss= 0.216879, Training Accuracy= 0.92969
Iter 56320, Minibatch Loss= 0.302488, Training Accuracy= 0.89844
Iter 57600, Minibatch Loss= 0.131714, Training Accuracy= 0.96094
Iter 58880, Minibatch Loss= 0.111539, Training Accuracy= 0.98438
Iter 60160, Minibatch Loss= 0.153051, Training Accuracy= 0.94531
Iter 61440, Minibatch Loss= 0.203167, Training Accuracy= 0.93750
Iter 62720, Minibatch Loss= 0.135487, Training Accuracy= 0.96875
Iter 64000, Minibatch Loss= 0.193364, Training Accuracy= 0.93750
Iter 65280, Minibatch Loss= 0.253839, Training Accuracy= 0.92969
Iter 66560, Minibatch Loss= 0.167367, Training Accuracy= 0.92969
Iter 67840, Minibatch Loss= 0.292977, Training Accuracy= 0.92188
Iter 69120, Minibatch Loss= 0.167282, Training Accuracy= 0.95312
Iter 70400, Minibatch Loss= 0.320158, Training Accuracy= 0.90625
Iter 71680, Minibatch Loss= 0.104524, Training Accuracy= 0.96875
Iter 72960, Minibatch Loss= 0.150813, Training Accuracy= 0.95312
Iter 74240, Minibatch Loss= 0.221704, Training Accuracy= 0.94531
Iter 75520, Minibatch Loss= 0.163385, Training Accuracy= 0.96094
Iter 76800, Minibatch Loss= 0.116737, Training Accuracy= 0.96875
Iter 78080, Minibatch Loss= 0.125065, Training Accuracy= 0.96094
Iter 79360, Minibatch Loss= 0.297941, Training Accuracy= 0.92188
Iter 80640, Minibatch Loss= 0.195890, Training Accuracy= 0.93750
Iter 81920, Minibatch Loss= 0.071490, Training Accuracy= 0.99219
Iter 83200, Minibatch Loss= 0.115205, Training Accuracy= 0.97656
Iter 84480, Minibatch Loss= 0.046660, Training Accuracy= 0.99219
Iter 85760, Minibatch Loss= 0.104727, Training Accuracy= 0.96875
Iter 87040, Minibatch Loss= 0.126642, Training Accuracy= 0.94531
Iter 88320, Minibatch Loss= 0.112083, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.084982, Training Accuracy= 0.98438
Iter 90880, Minibatch Loss= 0.184768, Training Accuracy= 0.94531
Iter 92160, Minibatch Loss= 0.192093, Training Accuracy= 0.94531
Iter 93440, Minibatch Loss= 0.108329, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.102838, Training Accuracy= 0.97656
Iter 96000, Minibatch Loss= 0.065941, Training Accuracy= 0.98438
Iter 97280, Minibatch Loss= 0.138307, Training Accuracy= 0.96094
Iter 98560, Minibatch Loss= 0.106404, Training Accuracy= 0.95312
Iter 99840, Minibatch Loss= 0.184734, Training Accuracy= 0.93750
Finished!
Testing Accuracy: 0.9765625
实例65：构建多层BiRNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
# dtype=tf.float32)
outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
dtype=tf.float32)
print(outputs[0].shape,len(outputs))
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.172426, Training Accuracy= 0.36719
Iter 2560, Minibatch Loss= 1.861094, Training Accuracy= 0.30469
Iter 3840, Minibatch Loss= 1.509905, Training Accuracy= 0.53125
Iter 5120, Minibatch Loss= 1.420893, Training Accuracy= 0.52344
Iter 6400, Minibatch Loss= 1.035572, Training Accuracy= 0.65625
Iter 7680, Minibatch Loss= 0.857329, Training Accuracy= 0.69531
Iter 8960, Minibatch Loss= 0.792173, Training Accuracy= 0.75000
Iter 10240, Minibatch Loss= 0.736671, Training Accuracy= 0.77344
Iter 11520, Minibatch Loss= 0.735905, Training Accuracy= 0.77344
Iter 12800, Minibatch Loss= 0.661815, Training Accuracy= 0.80469
Iter 14080, Minibatch Loss= 0.606804, Training Accuracy= 0.77344
Iter 15360, Minibatch Loss= 0.560650, Training Accuracy= 0.81250
Iter 16640, Minibatch Loss= 0.492422, Training Accuracy= 0.84375
Iter 17920, Minibatch Loss= 0.556491, Training Accuracy= 0.83594
Iter 19200, Minibatch Loss= 0.479821, Training Accuracy= 0.83594
Iter 20480, Minibatch Loss= 0.519516, Training Accuracy= 0.80469
Iter 21760, Minibatch Loss= 0.408189, Training Accuracy= 0.86719
Iter 23040, Minibatch Loss= 0.294251, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.237520, Training Accuracy= 0.92188
Iter 25600, Minibatch Loss= 0.510871, Training Accuracy= 0.87500
Iter 26880, Minibatch Loss= 0.335866, Training Accuracy= 0.89844
Iter 28160, Minibatch Loss= 0.234492, Training Accuracy= 0.92188
Iter 29440, Minibatch Loss= 0.294053, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.268390, Training Accuracy= 0.91406
Iter 32000, Minibatch Loss= 0.233152, Training Accuracy= 0.92188
Iter 33280, Minibatch Loss= 0.252517, Training Accuracy= 0.92969
Iter 34560, Minibatch Loss= 0.280433, Training Accuracy= 0.92188
Iter 35840, Minibatch Loss= 0.430729, Training Accuracy= 0.86719
Iter 37120, Minibatch Loss= 0.330445, Training Accuracy= 0.92969
Iter 38400, Minibatch Loss= 0.234638, Training Accuracy= 0.92969
Iter 39680, Minibatch Loss= 0.223448, Training Accuracy= 0.95312
Iter 40960, Minibatch Loss= 0.288984, Training Accuracy= 0.89844
Iter 42240, Minibatch Loss= 0.223206, Training Accuracy= 0.92969
Iter 43520, Minibatch Loss= 0.204564, Training Accuracy= 0.95312
Iter 44800, Minibatch Loss= 0.137583, Training Accuracy= 0.97656
Iter 46080, Minibatch Loss= 0.172719, Training Accuracy= 0.95312
Iter 47360, Minibatch Loss= 0.181917, Training Accuracy= 0.95312
Iter 48640, Minibatch Loss= 0.245667, Training Accuracy= 0.93750
Iter 49920, Minibatch Loss= 0.171087, Training Accuracy= 0.94531
Iter 51200, Minibatch Loss= 0.273686, Training Accuracy= 0.92969
Iter 52480, Minibatch Loss= 0.110942, Training Accuracy= 0.96094
Iter 53760, Minibatch Loss= 0.252516, Training Accuracy= 0.92969
Iter 55040, Minibatch Loss= 0.184445, Training Accuracy= 0.96094
Iter 56320, Minibatch Loss= 0.212692, Training Accuracy= 0.93750
Iter 57600, Minibatch Loss= 0.148188, Training Accuracy= 0.94531
Iter 58880, Minibatch Loss= 0.202650, Training Accuracy= 0.93750
Iter 60160, Minibatch Loss= 0.162564, Training Accuracy= 0.95312
Iter 61440, Minibatch Loss= 0.140252, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.080859, Training Accuracy= 0.98438
Iter 64000, Minibatch Loss= 0.271262, Training Accuracy= 0.92188
Iter 65280, Minibatch Loss= 0.130727, Training Accuracy= 0.96875
Iter 66560, Minibatch Loss= 0.202950, Training Accuracy= 0.94531
Iter 67840, Minibatch Loss= 0.104672, Training Accuracy= 0.97656
Iter 69120, Minibatch Loss= 0.104267, Training Accuracy= 0.96875
Iter 70400, Minibatch Loss= 0.264764, Training Accuracy= 0.91406
Iter 71680, Minibatch Loss= 0.247204, Training Accuracy= 0.92969
Iter 72960, Minibatch Loss= 0.114761, Training Accuracy= 0.96875
Iter 74240, Minibatch Loss= 0.127167, Training Accuracy= 0.96875
Iter 75520, Minibatch Loss= 0.150572, Training Accuracy= 0.95312
Iter 76800, Minibatch Loss= 0.266346, Training Accuracy= 0.92969
Iter 78080, Minibatch Loss= 0.188211, Training Accuracy= 0.92188
Iter 79360, Minibatch Loss= 0.178170, Training Accuracy= 0.94531
Iter 80640, Minibatch Loss= 0.065349, Training Accuracy= 0.99219
Iter 81920, Minibatch Loss= 0.114197, Training Accuracy= 0.96094
Iter 83200, Minibatch Loss= 0.166545, Training Accuracy= 0.94531
Iter 84480, Minibatch Loss= 0.170978, Training Accuracy= 0.96094
Iter 85760, Minibatch Loss= 0.100027, Training Accuracy= 0.98438
Iter 87040, Minibatch Loss= 0.068403, Training Accuracy= 0.97656
Iter 88320, Minibatch Loss= 0.084172, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.109292, Training Accuracy= 0.96875
Iter 90880, Minibatch Loss= 0.142447, Training Accuracy= 0.96094
Iter 92160, Minibatch Loss= 0.153235, Training Accuracy= 0.94531
Iter 93440, Minibatch Loss= 0.244075, Training Accuracy= 0.93750
Iter 94720, Minibatch Loss= 0.181213, Training Accuracy= 0.96094
Iter 96000, Minibatch Loss= 0.112537, Training Accuracy= 0.97656
Iter 97280, Minibatch Loss= 0.081839, Training Accuracy= 0.97656
Iter 98560, Minibatch Loss= 0.052867, Training Accuracy= 0.99219
Iter 99840, Minibatch Loss= 0.153948, Training Accuracy= 0.96875
Finished!
Testing Accuracy: 0.984375
list多层BiRNN
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
# dtype=tf.float32)
#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
# dtype=tf.float32)
stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):
stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,
dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.821590, Training Accuracy= 0.35156
Iter 2560, Minibatch Loss= 1.534426, Training Accuracy= 0.45312
Iter 3840, Minibatch Loss= 1.064070, Training Accuracy= 0.64062
Iter 5120, Minibatch Loss= 0.864886, Training Accuracy= 0.69531
Iter 6400, Minibatch Loss= 0.761668, Training Accuracy= 0.71094
Iter 7680, Minibatch Loss= 0.740874, Training Accuracy= 0.77344
Iter 8960, Minibatch Loss= 0.588481, Training Accuracy= 0.81250
Iter 10240, Minibatch Loss= 0.462982, Training Accuracy= 0.82812
Iter 11520, Minibatch Loss= 0.370633, Training Accuracy= 0.88281
Iter 12800, Minibatch Loss= 0.338352, Training Accuracy= 0.89062
Iter 14080, Minibatch Loss= 0.331004, Training Accuracy= 0.92188
Iter 15360, Minibatch Loss= 0.311066, Training Accuracy= 0.86719
Iter 16640, Minibatch Loss= 0.343783, Training Accuracy= 0.87500
Iter 17920, Minibatch Loss= 0.372595, Training Accuracy= 0.88281
Iter 19200, Minibatch Loss= 0.325485, Training Accuracy= 0.92188
Iter 20480, Minibatch Loss= 0.264375, Training Accuracy= 0.89844
Iter 21760, Minibatch Loss= 0.139555, Training Accuracy= 0.98438
Iter 23040, Minibatch Loss= 0.148688, Training Accuracy= 0.96094
Iter 24320, Minibatch Loss= 0.258720, Training Accuracy= 0.93750
Iter 25600, Minibatch Loss= 0.226550, Training Accuracy= 0.92188
Iter 26880, Minibatch Loss= 0.138458, Training Accuracy= 0.96875
Iter 28160, Minibatch Loss= 0.213251, Training Accuracy= 0.92188
Iter 29440, Minibatch Loss= 0.254001, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.167976, Training Accuracy= 0.96875
Iter 32000, Minibatch Loss= 0.135351, Training Accuracy= 0.95312
Iter 33280, Minibatch Loss= 0.160028, Training Accuracy= 0.94531
Iter 34560, Minibatch Loss= 0.077145, Training Accuracy= 0.98438
Iter 35840, Minibatch Loss= 0.191825, Training Accuracy= 0.94531
Iter 37120, Minibatch Loss= 0.165540, Training Accuracy= 0.95312
Iter 38400, Minibatch Loss= 0.196213, Training Accuracy= 0.94531
Iter 39680, Minibatch Loss= 0.287433, Training Accuracy= 0.93750
Iter 40960, Minibatch Loss= 0.191275, Training Accuracy= 0.95312
Iter 42240, Minibatch Loss= 0.151342, Training Accuracy= 0.96094
Iter 43520, Minibatch Loss= 0.125097, Training Accuracy= 0.96875
Iter 44800, Minibatch Loss= 0.102935, Training Accuracy= 0.98438
Iter 46080, Minibatch Loss= 0.266253, Training Accuracy= 0.92969
Iter 47360, Minibatch Loss= 0.142998, Training Accuracy= 0.95312
Iter 48640, Minibatch Loss= 0.184273, Training Accuracy= 0.93750
Iter 49920, Minibatch Loss= 0.119624, Training Accuracy= 0.95312
Iter 51200, Minibatch Loss= 0.142909, Training Accuracy= 0.96875
Iter 52480, Minibatch Loss= 0.116738, Training Accuracy= 0.96875
Iter 53760, Minibatch Loss= 0.060568, Training Accuracy= 0.97656
Iter 55040, Minibatch Loss= 0.058581, Training Accuracy= 0.98438
Iter 56320, Minibatch Loss= 0.124698, Training Accuracy= 0.96094
Iter 57600, Minibatch Loss= 0.100896, Training Accuracy= 0.98438
Iter 58880, Minibatch Loss= 0.110977, Training Accuracy= 0.95312
Iter 60160, Minibatch Loss= 0.106460, Training Accuracy= 0.97656
Iter 61440, Minibatch Loss= 0.110304, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.232378, Training Accuracy= 0.94531
Iter 64000, Minibatch Loss= 0.203506, Training Accuracy= 0.93750
Iter 65280, Minibatch Loss= 0.056710, Training Accuracy= 0.97656
Iter 66560, Minibatch Loss= 0.072753, Training Accuracy= 0.97656
Iter 67840, Minibatch Loss= 0.079578, Training Accuracy= 0.97656
Iter 69120, Minibatch Loss= 0.107077, Training Accuracy= 0.96094
Iter 70400, Minibatch Loss= 0.096304, Training Accuracy= 0.97656
Iter 71680, Minibatch Loss= 0.144363, Training Accuracy= 0.96094
Iter 72960, Minibatch Loss= 0.091259, Training Accuracy= 0.96094
Iter 74240, Minibatch Loss= 0.048933, Training Accuracy= 0.97656
Iter 75520, Minibatch Loss= 0.160002, Training Accuracy= 0.92969
Iter 76800, Minibatch Loss= 0.058088, Training Accuracy= 0.99219
Iter 78080, Minibatch Loss= 0.127986, Training Accuracy= 0.96094
Iter 79360, Minibatch Loss= 0.038077, Training Accuracy= 0.99219
Iter 80640, Minibatch Loss= 0.111614, Training Accuracy= 0.96875
Iter 81920, Minibatch Loss= 0.042832, Training Accuracy= 0.99219
Iter 83200, Minibatch Loss= 0.099700, Training Accuracy= 0.95312
Iter 84480, Minibatch Loss= 0.070158, Training Accuracy= 0.97656
Iter 85760, Minibatch Loss= 0.044101, Training Accuracy= 0.98438
Iter 87040, Minibatch Loss= 0.148444, Training Accuracy= 0.94531
Iter 88320, Minibatch Loss= 0.053288, Training Accuracy= 0.99219
Iter 89600, Minibatch Loss= 0.086438, Training Accuracy= 0.96875
Iter 90880, Minibatch Loss= 0.084410, Training Accuracy= 0.97656
Iter 92160, Minibatch Loss= 0.070647, Training Accuracy= 0.96875
Iter 93440, Minibatch Loss= 0.157281, Training Accuracy= 0.94531
Iter 94720, Minibatch Loss= 0.154711, Training Accuracy= 0.96875
Iter 96000, Minibatch Loss= 0.067240, Training Accuracy= 0.96875
Iter 97280, Minibatch Loss= 0.067891, Training Accuracy= 0.99219
Iter 98560, Minibatch Loss= 0.040220, Training Accuracy= 0.99219
Iter 99840, Minibatch Loss= 0.113512, Training Accuracy= 0.96094
Finished!
Testing Accuracy: 0.96875
MultiBiRNN
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
# dtype=tf.float32)
#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
# dtype=tf.float32)
stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):
stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,
# dtype=tf.float32)
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
mcell_bw = tf.contrib.rnn.MultiRNNCell(stacked_bw_rnn)
outputs, _, _ = rnn.stack_bidirectional_rnn([mcell],[mcell_bw], x1,
dtype=tf.float32)
#outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn([mcell],[mcell_bw], x,
# dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
#print(outputs[0].shape,outputs.shape)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.969089, Training Accuracy= 0.28906
Iter 2560, Minibatch Loss= 1.607973, Training Accuracy= 0.40625
Iter 3840, Minibatch Loss= 1.223222, Training Accuracy= 0.59375
Iter 5120, Minibatch Loss= 1.054859, Training Accuracy= 0.64844
Iter 6400, Minibatch Loss= 0.956632, Training Accuracy= 0.71875
Iter 7680, Minibatch Loss= 0.862249, Training Accuracy= 0.70312
Iter 8960, Minibatch Loss= 0.752958, Training Accuracy= 0.77344
Iter 10240, Minibatch Loss= 0.697205, Training Accuracy= 0.77344
Iter 11520, Minibatch Loss= 0.563148, Training Accuracy= 0.79688
Iter 12800, Minibatch Loss= 0.559524, Training Accuracy= 0.82031
Iter 14080, Minibatch Loss= 0.394891, Training Accuracy= 0.87500
Iter 15360, Minibatch Loss= 0.651662, Training Accuracy= 0.79688
Iter 16640, Minibatch Loss= 0.458168, Training Accuracy= 0.85938
Iter 17920, Minibatch Loss= 0.525461, Training Accuracy= 0.84375
Iter 19200, Minibatch Loss= 0.323415, Training Accuracy= 0.89844
Iter 20480, Minibatch Loss= 0.407945, Training Accuracy= 0.90625
Iter 21760, Minibatch Loss= 0.342875, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.278589, Training Accuracy= 0.92188
Iter 24320, Minibatch Loss= 0.248584, Training Accuracy= 0.91406
Iter 25600, Minibatch Loss= 0.314651, Training Accuracy= 0.88281
Iter 26880, Minibatch Loss= 0.256853, Training Accuracy= 0.90625
Iter 28160, Minibatch Loss= 0.305443, Training Accuracy= 0.92969
Iter 29440, Minibatch Loss= 0.370154, Training Accuracy= 0.90625
Iter 30720, Minibatch Loss= 0.262370, Training Accuracy= 0.92188
Iter 32000, Minibatch Loss= 0.239071, Training Accuracy= 0.92969
Iter 33280, Minibatch Loss= 0.277529, Training Accuracy= 0.90625
Iter 34560, Minibatch Loss= 0.147705, Training Accuracy= 0.96094
Iter 35840, Minibatch Loss= 0.210542, Training Accuracy= 0.92969
Iter 37120, Minibatch Loss= 0.221123, Training Accuracy= 0.92188
Iter 38400, Minibatch Loss= 0.357465, Training Accuracy= 0.92188
Iter 39680, Minibatch Loss= 0.164977, Training Accuracy= 0.95312
Iter 40960, Minibatch Loss= 0.243880, Training Accuracy= 0.92969
Iter 42240, Minibatch Loss= 0.213714, Training Accuracy= 0.94531
Iter 43520, Minibatch Loss= 0.133602, Training Accuracy= 0.96875
Iter 44800, Minibatch Loss= 0.304978, Training Accuracy= 0.90625
Iter 46080, Minibatch Loss= 0.228339, Training Accuracy= 0.92188
Iter 47360, Minibatch Loss= 0.240235, Training Accuracy= 0.92188
Iter 48640, Minibatch Loss= 0.154399, Training Accuracy= 0.96875
Iter 49920, Minibatch Loss= 0.252016, Training Accuracy= 0.92188
Iter 51200, Minibatch Loss= 0.171817, Training Accuracy= 0.96875
Iter 52480, Minibatch Loss= 0.145279, Training Accuracy= 0.96875
Iter 53760, Minibatch Loss= 0.164326, Training Accuracy= 0.93750
Iter 55040, Minibatch Loss= 0.099755, Training Accuracy= 0.96094
Iter 56320, Minibatch Loss= 0.085352, Training Accuracy= 0.97656
Iter 57600, Minibatch Loss= 0.172950, Training Accuracy= 0.95312
Iter 58880, Minibatch Loss= 0.123920, Training Accuracy= 0.96875
Iter 60160, Minibatch Loss= 0.115939, Training Accuracy= 0.97656
Iter 61440, Minibatch Loss= 0.084130, Training Accuracy= 0.97656
Iter 62720, Minibatch Loss= 0.149050, Training Accuracy= 0.93750
Iter 64000, Minibatch Loss= 0.168845, Training Accuracy= 0.95312
Iter 65280, Minibatch Loss= 0.140895, Training Accuracy= 0.96094
Iter 66560, Minibatch Loss= 0.126092, Training Accuracy= 0.95312
Iter 67840, Minibatch Loss= 0.147673, Training Accuracy= 0.96094
Iter 69120, Minibatch Loss= 0.057092, Training Accuracy= 0.99219
Iter 70400, Minibatch Loss= 0.037331, Training Accuracy= 1.00000
Iter 71680, Minibatch Loss= 0.188352, Training Accuracy= 0.95312
Iter 72960, Minibatch Loss= 0.108095, Training Accuracy= 0.96875
Iter 74240, Minibatch Loss= 0.059791, Training Accuracy= 0.99219
Iter 75520, Minibatch Loss= 0.113426, Training Accuracy= 0.96875
Iter 76800, Minibatch Loss= 0.133168, Training Accuracy= 0.97656
Iter 78080, Minibatch Loss= 0.183791, Training Accuracy= 0.95312
Iter 79360, Minibatch Loss= 0.122682, Training Accuracy= 0.96875
Iter 80640, Minibatch Loss= 0.057580, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.167665, Training Accuracy= 0.94531
Iter 83200, Minibatch Loss= 0.122606, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.091936, Training Accuracy= 0.98438
Iter 85760, Minibatch Loss= 0.171889, Training Accuracy= 0.95312
Iter 87040, Minibatch Loss= 0.117054, Training Accuracy= 0.96875
Iter 88320, Minibatch Loss= 0.051410, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.071451, Training Accuracy= 0.97656
Iter 90880, Minibatch Loss= 0.048790, Training Accuracy= 0.99219
Iter 92160, Minibatch Loss= 0.123331, Training Accuracy= 0.96094
Iter 93440, Minibatch Loss= 0.044237, Training Accuracy= 0.96875
Iter 94720, Minibatch Loss= 0.115570, Training Accuracy= 0.96875
Iter 96000, Minibatch Loss= 0.103702, Training Accuracy= 0.97656
Iter 97280, Minibatch Loss= 0.088024, Training Accuracy= 0.99219
Iter 98560, Minibatch Loss= 0.071973, Training Accuracy= 0.98438
Iter 99840, Minibatch Loss= 0.065105, Training Accuracy= 0.99219
Finished!
Testing Accuracy: 0.9765625
实例66：构建多层动态BiRNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
# dtype=tf.float32)
#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
# dtype=tf.float32)
stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):
stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,
# dtype=tf.float32)
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
mcell_bw = tf.contrib.rnn.MultiRNNCell(stacked_bw_rnn)
#outputs, _, _ = rnn.stack_bidirectional_rnn([mcell],[mcell_bw], x1,
# dtype=tf.float32)
outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn([mcell],[mcell_bw], x,
dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2]) #batch_size, max_time, layers_output]`
print(outputs[0].shape,outputs.shape)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.925515, Training Accuracy= 0.28906
Iter 2560, Minibatch Loss= 1.478539, Training Accuracy= 0.47656
Iter 3840, Minibatch Loss= 1.367934, Training Accuracy= 0.59375
Iter 5120, Minibatch Loss= 1.002120, Training Accuracy= 0.67188
Iter 6400, Minibatch Loss= 0.918952, Training Accuracy= 0.64062
Iter 7680, Minibatch Loss= 0.822888, Training Accuracy= 0.70312
Iter 8960, Minibatch Loss= 0.806084, Training Accuracy= 0.72656
Iter 10240, Minibatch Loss= 0.769249, Training Accuracy= 0.75781
Iter 11520, Minibatch Loss= 0.657592, Training Accuracy= 0.75781
Iter 12800, Minibatch Loss= 0.659502, Training Accuracy= 0.78125
Iter 14080, Minibatch Loss= 0.473548, Training Accuracy= 0.83594
Iter 15360, Minibatch Loss= 0.301996, Training Accuracy= 0.92188
Iter 16640, Minibatch Loss= 0.417549, Training Accuracy= 0.85938
Iter 17920, Minibatch Loss= 0.380066, Training Accuracy= 0.88281
Iter 19200, Minibatch Loss= 0.340269, Training Accuracy= 0.91406
Iter 20480, Minibatch Loss= 0.351742, Training Accuracy= 0.88281
Iter 21760, Minibatch Loss= 0.278537, Training Accuracy= 0.91406
Iter 23040, Minibatch Loss= 0.345716, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.338512, Training Accuracy= 0.92969
Iter 25600, Minibatch Loss= 0.301185, Training Accuracy= 0.92188
Iter 26880, Minibatch Loss= 0.335147, Training Accuracy= 0.90625
Iter 28160, Minibatch Loss= 0.329022, Training Accuracy= 0.89062
Iter 29440, Minibatch Loss= 0.278270, Training Accuracy= 0.90625
Iter 30720, Minibatch Loss= 0.141217, Training Accuracy= 0.96094
Iter 32000, Minibatch Loss= 0.316236, Training Accuracy= 0.91406
Iter 33280, Minibatch Loss= 0.170242, Training Accuracy= 0.96094
Iter 34560, Minibatch Loss= 0.169931, Training Accuracy= 0.96094
Iter 35840, Minibatch Loss= 0.261950, Training Accuracy= 0.91406
Iter 37120, Minibatch Loss= 0.275674, Training Accuracy= 0.92188
Iter 38400, Minibatch Loss= 0.269527, Training Accuracy= 0.92188
Iter 39680, Minibatch Loss= 0.288827, Training Accuracy= 0.92969
Iter 40960, Minibatch Loss= 0.123902, Training Accuracy= 0.97656
Iter 42240, Minibatch Loss= 0.222748, Training Accuracy= 0.93750
Iter 43520, Minibatch Loss= 0.275202, Training Accuracy= 0.91406
Iter 44800, Minibatch Loss= 0.114382, Training Accuracy= 0.97656
Iter 46080, Minibatch Loss= 0.232849, Training Accuracy= 0.93750
Iter 47360, Minibatch Loss= 0.183887, Training Accuracy= 0.96094
Iter 48640, Minibatch Loss= 0.143083, Training Accuracy= 0.94531
Iter 49920, Minibatch Loss= 0.119365, Training Accuracy= 0.95312
Iter 51200, Minibatch Loss= 0.165366, Training Accuracy= 0.93750
Iter 52480, Minibatch Loss= 0.142011, Training Accuracy= 0.96875
Iter 53760, Minibatch Loss= 0.124913, Training Accuracy= 0.96875
Iter 55040, Minibatch Loss= 0.271773, Training Accuracy= 0.91406
Iter 56320, Minibatch Loss= 0.151414, Training Accuracy= 0.94531
Iter 57600, Minibatch Loss= 0.312587, Training Accuracy= 0.89844
Iter 58880, Minibatch Loss= 0.128200, Training Accuracy= 0.96094
Iter 60160, Minibatch Loss= 0.082254, Training Accuracy= 0.96875
Iter 61440, Minibatch Loss= 0.143054, Training Accuracy= 0.94531
Iter 62720, Minibatch Loss= 0.189484, Training Accuracy= 0.94531
Iter 64000, Minibatch Loss= 0.145638, Training Accuracy= 0.95312
Iter 65280, Minibatch Loss= 0.095384, Training Accuracy= 0.96875
Iter 66560, Minibatch Loss= 0.107284, Training Accuracy= 0.96094
Iter 67840, Minibatch Loss= 0.072667, Training Accuracy= 0.98438
Iter 69120, Minibatch Loss= 0.078070, Training Accuracy= 0.98438
Iter 70400, Minibatch Loss= 0.145665, Training Accuracy= 0.96875
Iter 71680, Minibatch Loss= 0.131772, Training Accuracy= 0.96094
Iter 72960, Minibatch Loss= 0.054384, Training Accuracy= 0.97656
Iter 74240, Minibatch Loss= 0.080717, Training Accuracy= 0.99219
Iter 75520, Minibatch Loss= 0.127881, Training Accuracy= 0.94531
Iter 76800, Minibatch Loss= 0.050177, Training Accuracy= 0.98438
Iter 78080, Minibatch Loss= 0.204734, Training Accuracy= 0.92969
Iter 79360, Minibatch Loss= 0.134242, Training Accuracy= 0.95312
Iter 80640, Minibatch Loss= 0.102928, Training Accuracy= 0.96094
Iter 81920, Minibatch Loss= 0.151719, Training Accuracy= 0.96094
Iter 83200, Minibatch Loss= 0.063140, Training Accuracy= 0.98438
Iter 84480, Minibatch Loss= 0.044821, Training Accuracy= 0.99219
Iter 85760, Minibatch Loss= 0.042410, Training Accuracy= 0.99219
Iter 87040, Minibatch Loss= 0.043164, Training Accuracy= 1.00000
Iter 88320, Minibatch Loss= 0.095371, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.107568, Training Accuracy= 0.96094
Iter 90880, Minibatch Loss= 0.097504, Training Accuracy= 0.96875
Iter 92160, Minibatch Loss= 0.065621, Training Accuracy= 0.98438
Iter 93440, Minibatch Loss= 0.089349, Training Accuracy= 0.97656
Iter 94720, Minibatch Loss= 0.203302, Training Accuracy= 0.93750
Iter 96000, Minibatch Loss= 0.076798, Training Accuracy= 0.98438
Iter 97280, Minibatch Loss= 0.087679, Training Accuracy= 0.97656
Iter 98560, Minibatch Loss= 0.081388, Training Accuracy= 0.96875
Iter 99840, Minibatch Loss= 0.062613, Training Accuracy= 0.98438
Finished!
Testing Accuracy: 0.9765625
实例67：在GRUCell中实现LN
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
from tensorflow.python.ops.rnn_cell_impl import RNNCell
from tensorflow.python.ops.rnn_cell_impl import LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_ops
print(tf.__version__)
tf.reset_default_graph()
def ln(tensor, scope = None, epsilon = 1e-5):
""" Layer normalizes a 2D tensor along its second axis """
assert(len(tensor.get_shape()) == 2)
m, v = tf.nn.moments(tensor, [1], keep_dims=True)
if not isinstance(scope, str):
scope = ''
with tf.variable_scope(scope + 'layer_norm'):
scale = tf.get_variable('scale',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(1))
shift = tf.get_variable('shift',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(0))
LN_initial = (tensor - m) / tf.sqrt(v + epsilon)
return LN_initial * scale + shift
_BIAS_VARIABLE_NAME = "bias"
_WEIGHTS_VARIABLE_NAME = "kernel"
class LNGRUCell(LayerRNNCell):
"""Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
Args:
num_units: int, The number of units in the GRU cell.
activation: Nonlinearity to use. Default: `tanh`.
reuse: (optional) Python boolean describing whether to reuse variables
in an existing scope. If not `True`, and the existing scope already has
the given variables, an error is raised.
kernel_initializer: (optional) The initializer to use for the weight and
projection matrices.
bias_initializer: (optional) The initializer to use for the bias.
name: String, the name of the layer. Layers with the same name will
share weights, but to avoid mistakes we require reuse=True in such
cases.
"""
def __init__(self,
num_units,
activation=None,
reuse=None,
kernel_initializer=None,
bias_initializer=None,
name=None):
#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来
super(LNGRUCell, self).__init__(_reuse=reuse, name=name)
# Inputs must be 2-dimensional.
self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._activation = activation or math_ops.tanh
self._kernel_initializer = kernel_initializer
self._bias_initializer = bias_initializer
@property
def state_size(self):
return self._num_units
@property
def output_size(self):
return self._num_units
def build(self, inputs_shape):
if inputs_shape[1].value is None:
raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
% inputs_shape)
input_depth = inputs_shape[1].value
self._gate_kernel = self.add_variable(
"gates/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, 2 * self._num_units],
initializer=self._kernel_initializer)
self._gate_bias = self.add_variable(
"gates/%s" % _BIAS_VARIABLE_NAME,
shape=[2 * self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.constant_initializer(1.0, dtype=self.dtype)))
self._candidate_kernel = self.add_variable(
"candidate/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, self._num_units],
initializer=self._kernel_initializer)
self._candidate_bias = self.add_variable(
"candidate/%s" % _BIAS_VARIABLE_NAME,
shape=[self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.zeros_initializer(dtype=self.dtype)))
self.built = True
def call(self, inputs, state):
"""Gated recurrent unit (GRU) with nunits cells."""
gate_inputs = math_ops.matmul(
array_ops.concat([inputs, state], 1), self._gate_kernel)
gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)
value = math_ops.sigmoid(gate_inputs)
r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
r = ln(r, scope = 'r/')# 新添加
u = ln(u, scope = 'u/')# 新添加
r_state = r * state
candidate = math_ops.matmul(
array_ops.concat([inputs, r_state], 1), self._candidate_kernel)
candidate = nn_ops.bias_add(candidate, self._candidate_bias)
c = self._activation(candidate)
new_h = u * state + (1 - u) * c
return new_h, new_h
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
gru = LNGRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 12.231302, Training Accuracy= 0.08594
Iter 2560, Minibatch Loss= 11.580451, Training Accuracy= 0.08594
Iter 3840, Minibatch Loss= 12.903033, Training Accuracy= 0.06250
Iter 5120, Minibatch Loss= 10.890457, Training Accuracy= 0.12500
Iter 6400, Minibatch Loss= 9.708689, Training Accuracy= 0.12500
Iter 7680, Minibatch Loss= 11.344480, Training Accuracy= 0.07031
Iter 8960, Minibatch Loss= 21.113043, Training Accuracy= 0.11719
Iter 10240, Minibatch Loss= 16.354618, Training Accuracy= 0.12500
Iter 11520, Minibatch Loss= 221.128204, Training Accuracy= 0.15625
Iter 12800, Minibatch Loss= 51.676239, Training Accuracy= 0.14062
Iter 14080, Minibatch Loss= 51.621651, Training Accuracy= 0.16406
Iter 15360, Minibatch Loss= 32.979252, Training Accuracy= 0.13281
Iter 16640, Minibatch Loss= 11.387983, Training Accuracy= 0.16406
Iter 17920, Minibatch Loss= 106.375748, Training Accuracy= 0.13281
Iter 19200, Minibatch Loss= 162.647675, Training Accuracy= 0.07031
Iter 20480, Minibatch Loss= 70.351883, Training Accuracy= 0.03125
Iter 21760, Minibatch Loss= 55.518311, Training Accuracy= 0.12500
Iter 23040, Minibatch Loss= 31.811563, Training Accuracy= 0.10938
Iter 24320, Minibatch Loss= 42.223648, Training Accuracy= 0.11719
Iter 25600, Minibatch Loss= 278.500854, Training Accuracy= 0.08594
Iter 26880, Minibatch Loss= 237.631607, Training Accuracy= 0.10938
Iter 28160, Minibatch Loss= 105.564705, Training Accuracy= 0.14062
Iter 29440, Minibatch Loss= 56.833820, Training Accuracy= 0.08594
Iter 30720, Minibatch Loss= 22.568174, Training Accuracy= 0.10938
Iter 32000, Minibatch Loss= 11.444073, Training Accuracy= 0.10156
Iter 33280, Minibatch Loss= 5.311275, Training Accuracy= 0.11719
Iter 34560, Minibatch Loss= 3.783988, Training Accuracy= 0.10938
Iter 35840, Minibatch Loss= 5.724679, Training Accuracy= 0.10938
Iter 37120, Minibatch Loss= 13.915645, Training Accuracy= 0.06250
Iter 38400, Minibatch Loss= 7.799623, Training Accuracy= 0.06250
Iter 39680, Minibatch Loss= 4.735735, Training Accuracy= 0.10156
Iter 40960, Minibatch Loss= 3.492851, Training Accuracy= 0.14062
Iter 42240, Minibatch Loss= 2.750641, Training Accuracy= 0.12500
Iter 43520, Minibatch Loss= 3.131867, Training Accuracy= 0.03125
Iter 44800, Minibatch Loss= 12.454140, Training Accuracy= 0.14062
Iter 46080, Minibatch Loss= 34.983681, Training Accuracy= 0.11719
Iter 47360, Minibatch Loss= 30.050432, Training Accuracy= 0.11719
Iter 48640, Minibatch Loss= 89.305038, Training Accuracy= 0.14062
Iter 49920, Minibatch Loss= 717.047119, Training Accuracy= 0.04688
Iter 51200, Minibatch Loss= 593.221436, Training Accuracy= 0.07812
Iter 52480, Minibatch Loss= 552.920593, Training Accuracy= 0.07812
Iter 53760, Minibatch Loss= 298.492462, Training Accuracy= 0.10938
Iter 55040, Minibatch Loss= 293.587799, Training Accuracy= 0.12500
Iter 56320, Minibatch Loss= 90.123314, Training Accuracy= 0.08594
Iter 57600, Minibatch Loss= 131.756165, Training Accuracy= 0.11719
Iter 58880, Minibatch Loss= 90.683121, Training Accuracy= 0.10156
Iter 60160, Minibatch Loss= 56.682476, Training Accuracy= 0.09375
Iter 61440, Minibatch Loss= 64.690117, Training Accuracy= 0.07031
Iter 62720, Minibatch Loss= 73.732903, Training Accuracy= 0.12500
Iter 64000, Minibatch Loss= 39.817921, Training Accuracy= 0.18750
Iter 65280, Minibatch Loss= 26.479805, Training Accuracy= 0.03906
Iter 66560, Minibatch Loss= 28.650154, Training Accuracy= 0.03906
Iter 67840, Minibatch Loss= 32.062138, Training Accuracy= 0.07812
Iter 69120, Minibatch Loss= 24.480392, Training Accuracy= 0.10938
Iter 70400, Minibatch Loss= 18.083035, Training Accuracy= 0.14062
Iter 71680, Minibatch Loss= 33.831909, Training Accuracy= 0.08594
Iter 72960, Minibatch Loss= 31.264828, Training Accuracy= 0.17969
Iter 74240, Minibatch Loss= 24.374344, Training Accuracy= 0.12500
Iter 75520, Minibatch Loss= 32.843307, Training Accuracy= 0.09375
Iter 76800, Minibatch Loss= 20.493128, Training Accuracy= 0.05469
Iter 78080, Minibatch Loss= 21.611864, Training Accuracy= 0.16406
Iter 79360, Minibatch Loss= 25.270271, Training Accuracy= 0.17188
Iter 80640, Minibatch Loss= 25.224813, Training Accuracy= 0.14844
Iter 81920, Minibatch Loss= 27.151264, Training Accuracy= 0.08594
Iter 83200, Minibatch Loss= 24.466787, Training Accuracy= 0.06250
Iter 84480, Minibatch Loss= 26.366667, Training Accuracy= 0.21094
Iter 85760, Minibatch Loss= 19.625292, Training Accuracy= 0.18750
Iter 87040, Minibatch Loss= 18.934591, Training Accuracy= 0.17188
Iter 88320, Minibatch Loss= 32.218357, Training Accuracy= 0.10938
Iter 89600, Minibatch Loss= 46.415516, Training Accuracy= 0.12500
Iter 90880, Minibatch Loss= 30.490982, Training Accuracy= 0.11719
Iter 92160, Minibatch Loss= 30.738121, Training Accuracy= 0.16406
Iter 93440, Minibatch Loss= 25.382904, Training Accuracy= 0.10938
Iter 94720, Minibatch Loss= 21.190367, Training Accuracy= 0.11719
Iter 96000, Minibatch Loss= 32.771732, Training Accuracy= 0.10938
Iter 97280, Minibatch Loss= 19.819483, Training Accuracy= 0.11719
Iter 98560, Minibatch Loss= 13.052861, Training Accuracy= 0.15625
Iter 99840, Minibatch Loss= 21.311205, Training Accuracy= 0.15625
Finished!
Testing Accuracy: 0.140625
Ln多GRu1
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
from tensorflow.python.ops.rnn_cell_impl import RNNCell
from tensorflow.python.ops.rnn_cell_impl import LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_ops
tf.reset_default_graph()
def ln(tensor, scope = None, epsilon = 1e-5):
""" Layer normalizes a 2D tensor along its second axis """
assert(len(tensor.get_shape()) == 2)
m, v = tf.nn.moments(tensor, [1], keep_dims=True)
if not isinstance(scope, str):
scope = ''
with tf.variable_scope(scope + 'layer_norm'):
scale = tf.get_variable('scale',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(1))
shift = tf.get_variable('shift',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(0))
LN_initial = (tensor - m) / tf.sqrt(v + epsilon)
return LN_initial * scale + shift
class LNGRUCell(LayerRNNCell):
"""Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
Args:
num_units: int, The number of units in the GRU cell.
activation: Nonlinearity to use. Default: `tanh`.
reuse: (optional) Python boolean describing whether to reuse variables
in an existing scope. If not `True`, and the existing scope already has
the given variables, an error is raised.
kernel_initializer: (optional) The initializer to use for the weight and
projection matrices.
bias_initializer: (optional) The initializer to use for the bias.
name: String, the name of the layer. Layers with the same name will
share weights, but to avoid mistakes we require reuse=True in such
cases.
"""
def __init__(self,
num_units,
activation=None,
reuse=None,
kernel_initializer=None,
bias_initializer=None,
name=None):
#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来
super(LNGRUCell, self).__init__(_reuse=reuse, name=name)
# Inputs must be 2-dimensional.
self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._activation = activation or math_ops.tanh
self._kernel_initializer = kernel_initializer
self._bias_initializer = bias_initializer
@property
def state_size(self):
return self._num_units
@property
def output_size(self):
return self._num_units
def build(self, inputs_shape):
if inputs_shape[1].value is None:
raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
% inputs_shape)
input_depth = inputs_shape[1].value
self._gate_kernel = self.add_variable(
"gates/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, 2 * self._num_units],
initializer=self._kernel_initializer)
self._gate_bias = self.add_variable(
"gates/%s" % _BIAS_VARIABLE_NAME,
shape=[2 * self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.constant_initializer(1.0, dtype=self.dtype)))
self._candidate_kernel = self.add_variable(
"candidate/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, self._num_units],
initializer=self._kernel_initializer)
self._candidate_bias = self.add_variable(
"candidate/%s" % _BIAS_VARIABLE_NAME,
shape=[self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.zeros_initializer(dtype=self.dtype)))
self.built = True
def call(self, inputs, state):
"""Gated recurrent unit (GRU) with nunits cells."""
gate_inputs = math_ops.matmul(
array_ops.concat([inputs, state], 1), self._gate_kernel)
gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)
value = math_ops.sigmoid(gate_inputs)
r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
r = ln(r, scope = 'r/')# 新添加
u = ln(u, scope = 'u/')# 新添加
r_state = r * state
candidate = math_ops.matmul(
array_ops.concat([inputs, r_state], 1), self._candidate_kernel)
candidate = nn_ops.bias_add(candidate, self._candidate_bias)
c = self._activation(candidate)
new_h = u * state + (1 - u) * c
return new_h, new_h
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
stacked_rnn = []
for i in range(3):
stacked_rnn.append(LNGRUCell(n_hidden))
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
outputs,states = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序，取最后一个时序outputs[-1]=(?,256)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 11.682723, Training Accuracy= 0.08594
Iter 2560, Minibatch Loss= 12.917633, Training Accuracy= 0.13281
Iter 3840, Minibatch Loss= 11.147453, Training Accuracy= 0.14062
Iter 5120, Minibatch Loss= 11.501393, Training Accuracy= 0.10938
Iter 6400, Minibatch Loss= 11.329225, Training Accuracy= 0.14062
Iter 7680, Minibatch Loss= 12.902426, Training Accuracy= 0.12500
Iter 8960, Minibatch Loss= 14.787603, Training Accuracy= 0.12500
Iter 10240, Minibatch Loss= 14.239416, Training Accuracy= 0.11719
Iter 11520, Minibatch Loss= 11.831277, Training Accuracy= 0.14062
Iter 12800, Minibatch Loss= 13.174420, Training Accuracy= 0.12500
Iter 14080, Minibatch Loss= 10.532808, Training Accuracy= 0.07812
Iter 15360, Minibatch Loss= 21.689716, Training Accuracy= 0.11719
Iter 16640, Minibatch Loss= 18.296116, Training Accuracy= 0.10938
Iter 17920, Minibatch Loss= 12.612600, Training Accuracy= 0.14844
Iter 19200, Minibatch Loss= 8.998665, Training Accuracy= 0.09375
Iter 20480, Minibatch Loss= 7.482079, Training Accuracy= 0.13281
Iter 21760, Minibatch Loss= 5.310290, Training Accuracy= 0.20312
Iter 23040, Minibatch Loss= 11.217176, Training Accuracy= 0.10156
Iter 24320, Minibatch Loss= 10.468469, Training Accuracy= 0.11719
Iter 25600, Minibatch Loss= 9.238225, Training Accuracy= 0.09375
Iter 26880, Minibatch Loss= 9.928499, Training Accuracy= 0.10156
Iter 28160, Minibatch Loss= 11.853710, Training Accuracy= 0.05469
Iter 29440, Minibatch Loss= 11.066288, Training Accuracy= 0.03906
Iter 30720, Minibatch Loss= 8.290665, Training Accuracy= 0.14844
Iter 32000, Minibatch Loss= 7.704072, Training Accuracy= 0.14844
Iter 33280, Minibatch Loss= 7.526510, Training Accuracy= 0.13281
Iter 34560, Minibatch Loss= 10.274215, Training Accuracy= 0.08594
Iter 35840, Minibatch Loss= 9.874534, Training Accuracy= 0.07812
Iter 37120, Minibatch Loss= 7.862291, Training Accuracy= 0.07812
Iter 38400, Minibatch Loss= 7.622520, Training Accuracy= 0.10156
Iter 39680, Minibatch Loss= 5.845595, Training Accuracy= 0.13281
Iter 40960, Minibatch Loss= 4.240401, Training Accuracy= 0.14062
Iter 42240, Minibatch Loss= 3.776126, Training Accuracy= 0.11719
Iter 43520, Minibatch Loss= 4.534883, Training Accuracy= 0.08594
Iter 44800, Minibatch Loss= 4.593569, Training Accuracy= 0.08594
Iter 46080, Minibatch Loss= 4.776147, Training Accuracy= 0.06250
Iter 47360, Minibatch Loss= 4.990581, Training Accuracy= 0.07812
Iter 48640, Minibatch Loss= 4.928415, Training Accuracy= 0.08594
Iter 49920, Minibatch Loss= 4.748688, Training Accuracy= 0.09375
Iter 51200, Minibatch Loss= 2.824999, Training Accuracy= 0.14062
Iter 52480, Minibatch Loss= 2.998337, Training Accuracy= 0.15625
Iter 53760, Minibatch Loss= 3.980899, Training Accuracy= 0.12500
Iter 55040, Minibatch Loss= 3.022208, Training Accuracy= 0.17969
Iter 56320, Minibatch Loss= 3.015280, Training Accuracy= 0.13281
Iter 57600, Minibatch Loss= 2.676783, Training Accuracy= 0.13281
Iter 58880, Minibatch Loss= 6.211295, Training Accuracy= 0.10938
Iter 60160, Minibatch Loss= 4.801533, Training Accuracy= 0.14062
Iter 61440, Minibatch Loss= 7.826266, Training Accuracy= 0.08594
Iter 62720, Minibatch Loss= 4.575137, Training Accuracy= 0.08594
Iter 64000, Minibatch Loss= 4.202699, Training Accuracy= 0.11719
Iter 65280, Minibatch Loss= 5.797447, Training Accuracy= 0.04688
Iter 66560, Minibatch Loss= 3.881121, Training Accuracy= 0.12500
Iter 67840, Minibatch Loss= 3.441604, Training Accuracy= 0.11719
Iter 69120, Minibatch Loss= 3.097083, Training Accuracy= 0.10938
Iter 70400, Minibatch Loss= 3.482257, Training Accuracy= 0.09375
Iter 71680, Minibatch Loss= 3.060152, Training Accuracy= 0.09375
Iter 72960, Minibatch Loss= 2.767920, Training Accuracy= 0.14062
Iter 74240, Minibatch Loss= 2.690722, Training Accuracy= 0.10156
Iter 75520, Minibatch Loss= 2.394173, Training Accuracy= 0.08594
Iter 76800, Minibatch Loss= 2.383599, Training Accuracy= 0.11719
Iter 78080, Minibatch Loss= 1.973354, Training Accuracy= 0.23438
Iter 79360, Minibatch Loss= 2.104026, Training Accuracy= 0.23438
Iter 80640, Minibatch Loss= 2.073925, Training Accuracy= 0.21094
Iter 81920, Minibatch Loss= 1.942348, Training Accuracy= 0.25781
Iter 83200, Minibatch Loss= 2.008878, Training Accuracy= 0.26562
Iter 84480, Minibatch Loss= 1.999240, Training Accuracy= 0.28125
Iter 85760, Minibatch Loss= 1.956447, Training Accuracy= 0.23438
Iter 87040, Minibatch Loss= 1.900973, Training Accuracy= 0.25781
Iter 88320, Minibatch Loss= 2.097849, Training Accuracy= 0.25000
Iter 89600, Minibatch Loss= 2.013985, Training Accuracy= 0.28906
Iter 90880, Minibatch Loss= 1.828421, Training Accuracy= 0.28125
Iter 92160, Minibatch Loss= 1.944018, Training Accuracy= 0.29688
Iter 93440, Minibatch Loss= 1.839489, Training Accuracy= 0.33594
Iter 94720, Minibatch Loss= 1.945905, Training Accuracy= 0.28125
Iter 96000, Minibatch Loss= 2.672914, Training Accuracy= 0.10156
Iter 97280, Minibatch Loss= 2.854921, Training Accuracy= 0.03906
Iter 98560, Minibatch Loss= 2.538456, Training Accuracy= 0.04688
Iter 99840, Minibatch Loss= 2.212051, Training Accuracy= 0.19531
Finished!
Testing Accuracy: 0.24
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
from tensorflow.python.ops.rnn_cell_impl import RNNCell
from tensorflow.python.ops.rnn_cell_impl import LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_ops
tf.reset_default_graph()
print(tf.__version__)
def ln(tensor, scope = None, epsilon = 1e-5):
""" Layer normalizes a 2D tensor along its second axis """
assert(len(tensor.get_shape()) == 2)
m, v = tf.nn.moments(tensor, [1], keep_dims=True)
if not isinstance(scope, str):
scope = ''
with tf.variable_scope(scope + 'layer_norm'):
scale = tf.get_variable('scale',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(1))
shift = tf.get_variable('shift',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(0))
LN_initial = (tensor - m) / tf.sqrt(v + epsilon)
return LN_initial * scale + shift
#class LNGRUCell(RNNCell):
# """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078)."""
#
# def __init__(self, num_units, input_size=None, activation=tanh):
# if input_size is not None:
# print("%s: The input_size parameter is deprecated." % self)
# self._num_units = num_units
# self._activation = activation
#
# @property
# def state_size(self):
# return self._num_units
#
# @property
# def output_size(self):
# return self._num_units
#
# def __call__(self, inputs, state):
# """Gated recurrent unit (GRU) with nunits cells."""
# with vs.variable_scope("Gates"): # Reset gate and update gate.,reuse=True
# # We start with bias of 1.0 to not reset and not update.
# value =_linear([inputs, state], 2 * self._num_units, True, 1.0)
# r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
# r = ln(r, scope = 'r/')
# u = ln(u, scope = 'u/')
# r, u = sigmoid(r), sigmoid(u)
# with vs.variable_scope("Candidate"):
## with vs.variable_scope("Layer_Parameters"):
# Cand = _linear([inputs, r *state], self._num_units, True)
# c_pre = ln(Cand, scope = 'new_h/')
# c = self._activation(c_pre)
# new_h = u * state + (1 - u) * c
# return new_h, new_h
class LNGRUCell(LayerRNNCell):
"""
Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
Args:
num_units: int, The number of units in the GRU cell.
activation: Nonlinearity to use. Default: `tanh`.
reuse: (optional) Python boolean describing whether to reuse variables
in an existing scope. If not `True`, and the existing scope already has
the given variables, an error is raised.
kernel_initializer: (optional) The initializer to use for the weight and
projection matrices.
bias_initializer: (optional) The initializer to use for the bias.
name: String, the name of the layer. Layers with the same name will
share weights, but to avoid mistakes we require reuse=True in such
cases.
"""
def __init__(self,
num_units,
activation=None,
reuse=None,
kernel_initializer=None,
bias_initializer=None,
name=None):
#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来
super(LNGRUCell, self).__init__(_reuse=reuse, name=name)
# Inputs must be 2-dimensional.
self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._activation = activation or math_ops.tanh
self._kernel_initializer = kernel_initializer
self._bias_initializer = bias_initializer
@property
def state_size(self):
return self._num_units
@property
def output_size(self):
return self._num_units
def build(self, inputs_shape):
if inputs_shape[1].value is None:
raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
% inputs_shape)
input_depth = inputs_shape[1].value
self._gate_kernel = self.add_variable(
"gates/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, 2 * self._num_units],
initializer=self._kernel_initializer)
self._gate_bias = self.add_variable(
"gates/%s" % _BIAS_VARIABLE_NAME,
shape=[2 * self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.constant_initializer(1.0, dtype=self.dtype)))
self._candidate_kernel = self.add_variable(
"candidate/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, self._num_units],
initializer=self._kernel_initializer)
self._candidate_bias = self.add_variable(
"candidate/%s" % _BIAS_VARIABLE_NAME,
shape=[self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.zeros_initializer(dtype=self.dtype)))
self.built = True
def call(self, inputs, state):
"""Gated recurrent unit (GRU) with nunits cells."""
gate_inputs = math_ops.matmul(
array_ops.concat([inputs, state], 1), self._gate_kernel)
gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)
value = math_ops.sigmoid(gate_inputs)
r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
r = ln(r, scope = 'r/')# 新添加
u = ln(u, scope = 'u/')# 新添加
r_state = r * state
candidate = math_ops.matmul(
array_ops.concat([inputs, r_state], 1), self._candidate_kernel)
candidate = nn_ops.bias_add(candidate, self._candidate_bias)
c = self._activation(candidate)
new_h = u * state + (1 - u) * c
return new_h, new_h
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ，一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
stacked_rnn = []
for i in range(3):
stacked_rnn.append(LNGRUCell(n_hidden))
#stacked_rnn.append(tf.contrib.rnn.GRUCell(n_hidden))
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
#outputs,states = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
#outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序，取最后一个时序outputs[-1]=(?,256)
#pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
#
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 12.811991, Training Accuracy= 0.09375
Iter 2560, Minibatch Loss= 12.545067, Training Accuracy= 0.07031
Iter 3840, Minibatch Loss= 11.355438, Training Accuracy= 0.07031
Iter 5120, Minibatch Loss= 10.871563, Training Accuracy= 0.10156
Iter 6400, Minibatch Loss= 11.371557, Training Accuracy= 0.11719
Iter 7680, Minibatch Loss= 11.038004, Training Accuracy= 0.10938
Iter 8960, Minibatch Loss= 11.118206, Training Accuracy= 0.07812
Iter 10240, Minibatch Loss= 10.915785, Training Accuracy= 0.10156
Iter 11520, Minibatch Loss= 10.551557, Training Accuracy= 0.17969
Iter 12800, Minibatch Loss= 17.443115, Training Accuracy= 0.07031
Iter 14080, Minibatch Loss= 9.930510, Training Accuracy= 0.16406
Iter 15360, Minibatch Loss= 10.727852, Training Accuracy= 0.08594
Iter 16640, Minibatch Loss= 14.488325, Training Accuracy= 0.10938
Iter 17920, Minibatch Loss= 15.069497, Training Accuracy= 0.10938
Iter 19200, Minibatch Loss= 9.160069, Training Accuracy= 0.10938
Iter 20480, Minibatch Loss= 11.217492, Training Accuracy= 0.06250
Iter 21760, Minibatch Loss= 9.194048, Training Accuracy= 0.07812
Iter 23040, Minibatch Loss= 6.942473, Training Accuracy= 0.06250
Iter 24320, Minibatch Loss= 7.054324, Training Accuracy= 0.10938
Iter 25600, Minibatch Loss= 7.900481, Training Accuracy= 0.06250
Iter 26880, Minibatch Loss= 7.393861, Training Accuracy= 0.14844
Iter 28160, Minibatch Loss= 6.575096, Training Accuracy= 0.12500
Iter 29440, Minibatch Loss= 6.346336, Training Accuracy= 0.08594
Iter 30720, Minibatch Loss= 6.810472, Training Accuracy= 0.11719
Iter 32000, Minibatch Loss= 6.466897, Training Accuracy= 0.10938
Iter 33280, Minibatch Loss= 5.635717, Training Accuracy= 0.11719
Iter 34560, Minibatch Loss= 4.871469, Training Accuracy= 0.07812
Iter 35840, Minibatch Loss= 4.517097, Training Accuracy= 0.12500
Iter 37120, Minibatch Loss= 4.759137, Training Accuracy= 0.13281
Iter 38400, Minibatch Loss= 4.027944, Training Accuracy= 0.13281
Iter 39680, Minibatch Loss= 4.569145, Training Accuracy= 0.10156
Iter 40960, Minibatch Loss= 2.790155, Training Accuracy= 0.13281
Iter 42240, Minibatch Loss= 3.320386, Training Accuracy= 0.14844
Iter 43520, Minibatch Loss= 3.730759, Training Accuracy= 0.11719
Iter 44800, Minibatch Loss= 6.588100, Training Accuracy= 0.10938
Iter 46080, Minibatch Loss= 3.605313, Training Accuracy= 0.06250
Iter 47360, Minibatch Loss= 3.313602, Training Accuracy= 0.08594
Iter 48640, Minibatch Loss= 2.854012, Training Accuracy= 0.12500
Iter 49920, Minibatch Loss= 3.053122, Training Accuracy= 0.09375
Iter 51200, Minibatch Loss= 3.263433, Training Accuracy= 0.07812
Iter 52480, Minibatch Loss= 3.164253, Training Accuracy= 0.10156
Iter 53760, Minibatch Loss= 2.568920, Training Accuracy= 0.08594
Iter 55040, Minibatch Loss= 2.622833, Training Accuracy= 0.08594
Iter 56320, Minibatch Loss= 2.574237, Training Accuracy= 0.10156
Iter 57600, Minibatch Loss= 2.320722, Training Accuracy= 0.14062
Iter 58880, Minibatch Loss= 2.416647, Training Accuracy= 0.14062
Iter 60160, Minibatch Loss= 2.378407, Training Accuracy= 0.12500
Iter 61440, Minibatch Loss= 2.236217, Training Accuracy= 0.21094
Iter 62720, Minibatch Loss= 2.288638, Training Accuracy= 0.15625
Iter 64000, Minibatch Loss= 2.273193, Training Accuracy= 0.16406
Iter 65280, Minibatch Loss= 2.243811, Training Accuracy= 0.16406
Iter 66560, Minibatch Loss= 2.228886, Training Accuracy= 0.21875
Iter 67840, Minibatch Loss= 2.233191, Training Accuracy= 0.19531
Iter 69120, Minibatch Loss= 2.175747, Training Accuracy= 0.21875
Iter 70400, Minibatch Loss= 2.162902, Training Accuracy= 0.23438
Iter 71680, Minibatch Loss= 2.325285, Training Accuracy= 0.14062
Iter 72960, Minibatch Loss= 2.281687, Training Accuracy= 0.19531
Iter 74240, Minibatch Loss= 2.340919, Training Accuracy= 0.14062
Iter 75520, Minibatch Loss= 2.432267, Training Accuracy= 0.12500
Iter 76800, Minibatch Loss= 2.398518, Training Accuracy= 0.13281
Iter 78080, Minibatch Loss= 2.573318, Training Accuracy= 0.10938
Iter 79360, Minibatch Loss= 2.431727, Training Accuracy= 0.15625
Iter 80640, Minibatch Loss= 2.496175, Training Accuracy= 0.10156
Iter 81920, Minibatch Loss= 2.485367, Training Accuracy= 0.10156
Iter 83200, Minibatch Loss= 2.326829, Training Accuracy= 0.09375
Iter 84480, Minibatch Loss= 2.413650, Training Accuracy= 0.14062
Iter 85760, Minibatch Loss= 2.327744, Training Accuracy= 0.13281
Iter 87040, Minibatch Loss= 2.327587, Training Accuracy= 0.13281
Iter 88320, Minibatch Loss= 2.335200, Training Accuracy= 0.11719
Iter 89600, Minibatch Loss= 2.370470, Training Accuracy= 0.15625
Iter 90880, Minibatch Loss= 2.429362, Training Accuracy= 0.09375
Iter 92160, Minibatch Loss= 2.286218, Training Accuracy= 0.12500
Iter 93440, Minibatch Loss= 2.334179, Training Accuracy= 0.14844
Iter 94720, Minibatch Loss= 2.313130, Training Accuracy= 0.15625
Iter 96000, Minibatch Loss= 2.388861, Training Accuracy= 0.10938
Iter 97280, Minibatch Loss= 2.468186, Training Accuracy= 0.16406
Iter 98560, Minibatch Loss= 2.380651, Training Accuracy= 0.12500
Iter 99840, Minibatch Loss= 2.311615, Training Accuracy= 0.08594
Finished!
Testing Accuracy: 0.14
实例68：利用BiRNN实现语音识别
# -*- coding: utf-8 -*-
import numpy as np
import time
import tensorflow as tf
from tensorflow.python.ops import ctc_ops
from collections import Counter
## 自定义
yuyinutils = __import__("9-24__yuyinutils")
sparse_tuple_to_texts_ch = yuyinutils.sparse_tuple_to_texts_ch
ndarray_to_text_ch = yuyinutils.ndarray_to_text_ch
get_audio_and_transcriptch = yuyinutils.get_audio_and_transcriptch
pad_sequences = yuyinutils.pad_sequences
sparse_tuple_from = yuyinutils.sparse_tuple_from
get_wavs_lables = yuyinutils.get_wavs_lables
tf.reset_default_graph()
b_stddev = 0.046875
h_stddev = 0.046875
n_hidden = 1024
n_hidden_1 = 1024
n_hidden_2 =1024
n_hidden_5 = 1024
n_cell_dim = 1024
n_hidden_3 = 2 * 1024
keep_dropout_rate=0.95
relu_clip = 20
def BiRNN_model( batch_x, seq_length, n_input, n_context,n_character ,keep_dropout):
# batch_x_shape: [batch_size, n_steps, n_input + 2*n_input*n_context]
batch_x_shape = tf.shape(batch_x)
# 将输入转成时间序列优先
batch_x = tf.transpose(batch_x, [1, 0, 2])
# 再转成2维传入第一层
batch_x = tf.reshape(batch_x,
[-1, n_input + 2 * n_input * n_context]) # (n_steps*batch_size, n_input + 2*n_input*n_context)
# 使用clipped RELU activation and dropout.
# 1st layer
with tf.name_scope('fc1'):
b1 = variable_on_cpu('b1', [n_hidden_1], tf.random_normal_initializer(stddev=b_stddev))
h1 = variable_on_cpu('h1', [n_input + 2 * n_input * n_context, n_hidden_1],
tf.random_normal_initializer(stddev=h_stddev))
layer_1 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), relu_clip)
layer_1 = tf.nn.dropout(layer_1, keep_dropout)
# 2nd layer
with tf.name_scope('fc2'):
b2 = variable_on_cpu('b2', [n_hidden_2], tf.random_normal_initializer(stddev=b_stddev))
h2 = variable_on_cpu('h2', [n_hidden_1, n_hidden_2], tf.random_normal_initializer(stddev=h_stddev))
layer_2 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), relu_clip)
layer_2 = tf.nn.dropout(layer_2, keep_dropout)
# 3rd layer
with tf.name_scope('fc3'):
b3 = variable_on_cpu('b3', [n_hidden_3], tf.random_normal_initializer(stddev=b_stddev))
h3 = variable_on_cpu('h3', [n_hidden_2, n_hidden_3], tf.random_normal_initializer(stddev=h_stddev))
layer_3 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), relu_clip)
layer_3 = tf.nn.dropout(layer_3, keep_dropout)
# 双向rnn
with tf.name_scope('lstm'):
# Forward direction cell:
lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)
lstm_fw_cell = tf.contrib.rnn.DropoutWrapper(lstm_fw_cell,
input_keep_prob=keep_dropout)
# Backward direction cell:
lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)
lstm_bw_cell = tf.contrib.rnn.DropoutWrapper(lstm_bw_cell,
input_keep_prob=keep_dropout)
# `layer_3` `[n_steps, batch_size, 2*n_cell_dim]`
layer_3 = tf.reshape(layer_3, [-1, batch_x_shape[0], n_hidden_3])
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
cell_bw=lstm_bw_cell,
inputs=layer_3,
dtype=tf.float32,
time_major=True,
sequence_length=seq_length)
# 连接正反向结果[n_steps, batch_size, 2*n_cell_dim]
outputs = tf.concat(outputs, 2)
# to a single tensor of shape [n_steps*batch_size, 2*n_cell_dim]
outputs = tf.reshape(outputs, [-1, 2 * n_cell_dim])
with tf.name_scope('fc5'):
b5 = variable_on_cpu('b5', [n_hidden_5], tf.random_normal_initializer(stddev=b_stddev))
h5 = variable_on_cpu('h5', [(2 * n_cell_dim), n_hidden_5], tf.random_normal_initializer(stddev=h_stddev))
layer_5 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(outputs, h5), b5)), relu_clip)
layer_5 = tf.nn.dropout(layer_5, keep_dropout)
with tf.name_scope('fc6'):
# 全连接层用于softmax分类
b6 = variable_on_cpu('b6', [n_character], tf.random_normal_initializer(stddev=b_stddev))
h6 = variable_on_cpu('h6', [n_hidden_5, n_character], tf.random_normal_initializer(stddev=h_stddev))
layer_6 = tf.add(tf.matmul(layer_5, h6), b6)
# 将2维[n_steps*batch_size, n_character]转成3维 time-major [n_steps, batch_size, n_character].
layer_6 = tf.reshape(layer_6, [-1, batch_x_shape[0], n_character])
# Output shape: [n_steps, batch_size, n_character]
return layer_6
"""
used to create a variable in CPU memory.
"""
def variable_on_cpu(name, shape, initializer):
# Use the /cpu:0 device for scoped operations
with tf.device('/cpu:0'):
# Create or get apropos variable
var = tf.get_variable(name=name, shape=shape, initializer=initializer)
return var
wav_path='H:/tensorflow_projects/chap9/data_thchs30/wav/train'
label_file='H:/tensorflow_projects/chap9/data_thchs30/doc/trans/train.word.txt'
wav_files, labels = get_wavs_lables(wav_path,label_file)
print(wav_files[0], labels[0])
# wav/train/A11/A11_0.WAV -> 绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然
print("wav:",len(wav_files),"label",len(labels))
# 字表
all_words = []
for label in labels:
#print(label)
all_words += [word for word in label]
counter = Counter(all_words)
words = sorted(counter)
words_size= len(words)
word_num_map = dict(zip(words, range(words_size)))
print('字表大小:', words_size)
n_input = 26#计算美尔倒谱系数的个数
n_context = 9#对于每个时间点，要包含上下文样本的个数
batch_size =8
def next_batch(labels, start_idx = 0,batch_size=1,wav_files = wav_files):
filesize = len(labels)
end_idx = min(filesize, start_idx + batch_size)
idx_list = range(start_idx, end_idx)
txt_labels = [labels[i] for i in idx_list]
wav_files = [wav_files[i] for i in idx_list]
(source, audio_len, target, transcript_len) = get_audio_and_transcriptch(None,
wav_files,
n_input,
n_context,word_num_map,txt_labels)
start_idx += batch_size
# Verify that the start_idx is not larger than total available sample size
if start_idx >= filesize:
start_idx = -1
# Pad input to max_time_step of this batch
source, source_lengths = pad_sequences(source)#如果多个文件将长度统一，支持按最大截断或补0
sparse_labels = sparse_tuple_from(target)
return start_idx,source, source_lengths, sparse_labels
next_idx,source,source_len,sparse_lab = next_batch(labels,0,batch_size)
print(len(sparse_lab))
print(np.shape(source))
#print(sparse_lab)
t = sparse_tuple_to_texts_ch(sparse_lab,words)
print(t[0])
#source已经将变为前9（不够补空）+本身+后9，每个26，第一个顺序是第10个的数据。
# shape = [batch_size, max_stepsize, n_input + (2 * n_input * n_context)]
# the batch_size and max_stepsize每步都是变长的。
input_tensor = tf.placeholder(tf.float32, [None, None, n_input + (2 * n_input * n_context)], name='input')#语音log filter bank or MFCC features
# Use sparse_placeholder; will generate a SparseTensor, required by ctc_loss op.
targets = tf.sparse_placeholder(tf.int32, name='targets')#文本
# 1d array of size [batch_size]
seq_length = tf.placeholder(tf.int32, [None], name='seq_length')#序列长
keep_dropout= tf.placeholder(tf.float32)
# logits is the non-normalized output/activations from the last layer.
# logits will be input for the loss function.
# nn_model is from the import statement in the load_model function
logits = BiRNN_model( input_tensor, tf.to_int64(seq_length), n_input, n_context,words_size +1,keep_dropout)
#调用ctc loss
avg_loss = tf.reduce_mean(ctc_ops.ctc_loss(targets, logits, seq_length))
#[optimizer]
learning_rate = 0.001
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(avg_loss)
with tf.name_scope("decode"):
decoded, log_prob = ctc_ops.ctc_beam_search_decoder( logits, seq_length, merge_repeated=False)
with tf.name_scope("accuracy"):
distance = tf.edit_distance( tf.cast(decoded[0], tf.int32), targets)
# 计算label error rate (accuracy)
ler = tf.reduce_mean(distance, name='label_error_rate')
epochs = 100
savedir = "log/yuyinchalltest/"
saver = tf.train.Saver(max_to_keep=1) # 生成saver
# create the session
sess = tf.Session()
# 没有模型的话，就重新初始化
sess.run(tf.global_variables_initializer())
kpt = tf.train.latest_checkpoint(savedir)
print("kpt:",kpt)
startepo= 0
if kpt!=None:
saver.restore(sess, kpt)
ind = kpt.find("-")
startepo = int(kpt[ind+1:])
print(startepo)
# 准备运行训练步骤
section = '\n{0:=^40}\n'
print(section.format('Run training epoch'))
train_start = time.time()
for epoch in range(epochs):#样本集迭代次数
epoch_start = time.time()
if epoch<startepo:
continue
print("epoch start:",epoch,"total epochs= ",epochs)
#######################run batch####
n_batches_per_epoch = int(np.ceil(len(labels) / batch_size))
print("total loop ",n_batches_per_epoch,"in one epoch，",batch_size,"items in one loop")
train_cost = 0
train_ler = 0
next_idx =0
for batch in range(n_batches_per_epoch):#一次batch_size，取多少次
#取数据
next_idx,source,source_lengths,sparse_labels = \
next_batch(labels,next_idx ,batch_size)
feed = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:keep_dropout_rate}
#计算 avg_loss optimizer ;
batch_cost, _ = sess.run([avg_loss, optimizer], feed_dict=feed )
train_cost += batch_cost
if (batch +1)%20 == 0:
print('loop:',batch, 'Train cost: ', train_cost/(batch+1))
feed2 = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:1.0}
d,train_ler = sess.run([decoded[0],ler], feed_dict=feed2)
dense_decoded = tf.sparse_tensor_to_dense( d, default_value=-1).eval(session=sess)
dense_labels = sparse_tuple_to_texts_ch(sparse_labels,words)
counter =0
print('Label err rate: ', train_ler)
for orig, decoded_arr in zip(dense_labels, dense_decoded):
# convert to strings
decoded_str = ndarray_to_text_ch(decoded_arr,words)
print(' file {}'.format( counter))
print('Original: {}'.format(orig))
print('Decoded: {}'.format(decoded_str))
counter=counter+1
break
epoch_duration = time.time() - epoch_start
log = 'Epoch {}/{}, train_cost: {:.3f}, train_ler: {:.3f}, time: {:.2f} sec'
print(log.format(epoch ,epochs, train_cost,train_ler,epoch_duration))
saver.save(sess, savedir+"yuyinch.cpkt", global_step=epoch)
train_duration = time.time() - train_start
print('Training complete, total duration: {:.2f} min'.format(train_duration / 60))
sess.close()
H:/tensorflow_projects/chap9/data_thchs30/wav/train\A11\A11_0.WAV 绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然
wav: 8911 label 8911
字表大小: 2666
3
(8, 584, 494)
绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然
WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:197: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:53: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
* https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
* https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.
WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:73: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.
WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:89: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
kpt: None
===========Run training epoch===========
epoch start: 0 total epochs= 100
total loop 1114 in one epoch， 8 items in one loop
loop: 19 Train cost: 858.0828048706055
Label err rate: 0.98041475
file 0
Original: 小 鸳鸯 成长 很快 到了 深秋 在 北方 出生 的 小 鸳鸯 便 能 跟随 鸳鸯 大群 一起 南下 越冬 了
Decoded:
loop: 39 Train cost: 575.0958625793457
Label err rate: 0.9796773
file 0
Original: 马 晓 年 作画 的 特征 与 神韵 其 画风 区别 于 文人 画 和 年画 可谓 雅俗共赏
Decoded:
loop: 59 Train cost: 475.83005854288734
Label err rate: 0.97848856
file 0
Original: 另外 加工 修理 和 修配 业务 不 属于 营业税 的 应 税 劳务 不 缴纳 营业税
Decoded:
loop: 79 Train cost: 422.05160999298096
Label err rate: 0.9786789
file 0
Original: 这 碗 离 娘 饭 姑娘 再有 离 娘 痛楚 也 要 每样 都 吃 一点 才 算 循 规 遵 俗 的
Decoded:
loop: 99 Train cost: 389.91693817138673
Label err rate: 0.97975487
file 0
Original: 国防大学 政委 王茂 润 中将 与 延安 八一 敬老院 八 十五岁 的 老红军 柴 福耀 亲切 交谈
Decoded:
loop: 119 Train cost: 368.8437096913656
Label err rate: 0.973868
file 0
Original: 仅 绘画 而论 齐白石 是 巍巍 昆仑 可 这位 附庸风雅 的 门外汉 连 一块 石头 都 不是
Decoded: 的
loop: 139 Train cost: 353.2190541948591
Label err rate: 0.95776415
file 0
Original: 这次 全国 青年 排球 联赛 共 设 天津 舟山 武汉 三个 赛区 每个 赛区 的 前 两 名将 参加 复赛
Decoded: 的
loop: 159 Train cost: 341.92204132080076
Label err rate: 0.9404359
file 0
Original: 人代会 开幕 那天 奥 申 委 执行主席 伍 绍 祖 在 人民大会堂 东 大厅 一 露面 就 被 团团 围住
Decoded: 的 龚龚
loop: 179 Train cost: 333.7818296644423
Label err rate: 0.9725291
file 0
Original: 主要原因 是 世界 红 白藤 主产 国 印尼 禁止 红 白藤 原料 出口 导致 国际 红 白藤 原料 价格 上涨
Decoded: 龚龚
loop: 199 Train cost: 326.5086625671387
Label err rate: 0.93530095
file 0
Original: 赛 距 为 职业 运动员 最长 二十四 千米 业余 运动员 最长 二十一 千米 青年 运动员 最长 十五 千米
Decoded: 的 龚龚
loop: 219 Train cost: 321.0388072620739
Label err rate: 0.980114
file 0
Original: 据说 母亲 生下 没 一年 功夫 染 上了 一 种 怪病 不吃 不喝 乱 哭 乱 闹 急 得 外婆 满山遍野 去找 草药 来 驱鬼
Decoded:
loop: 239 Train cost: 316.15502440134685
Label err rate: 0.9510975
file 0
Original: 所谓 软 是指 软件 包括 专利 使用权 设计 图纸 技术 诀窍 纯 技术 引进 不 带 硬件
Decoded: 的
loop: 259 Train cost: 311.4005855266864
Label err rate: 0.9620612
file 0
Original: 要知道 秦俑 二 号 坑 的 军 阵 性质 首先 要 了解一下 学者 们 关于 秦俑 一号 坑 军 阵 性质 的 探讨
Decoded: 龚龚
loop: 279 Train cost: 307.48678610665456
Label err rate: 0.9382373
file 0
Original: 使用 这样的 牙刷 会 不同 程度 地 刺伤 牙龈 造成 牙龈出血 并 导致 牙齿 过度 磨损
Decoded: 的 龚龚
loop: 299 Train cost: 304.18842356363933
Label err rate: 0.9661161
file 0
Original: 新华社 基 多 二月 六 日电 最近 二 十天 来 厄瓜多尔 全国 疟疾 迅速 蔓延 已有 多 人 死亡
Decoded: 的
loop: 319 Train cost: 301.10915036201476
实例69：利用RNN训练语言模型
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import random
import time
from collections import Counter
start_time = time.time()
def elapsed(sec):
if sec<60:
return str(sec) + " sec"
elif sec<(60*60):
return str(sec/60) + " min"
else:
return str(sec/(60*60)) + " hr"
# Target log path
tf.reset_default_graph()
training_file = 'wordstest.txt'
#中文多文件
def readalltxt(txt_files):
labels = []
for txt_file in txt_files:
target = get_ch_lable(txt_file)
labels.append(target)
return labels
#中文字
def get_ch_lable(txt_file):
labels= ""
with open(txt_file, 'rb') as f:
for label in f:
#labels =label.decode('utf-8')
labels =labels+label.decode('gb2312')
return labels
#优先转文件里的字符到向量
def get_ch_lable_v(txt_file,word_num_map,txt_label=None):
words_size = len(word_num_map)
to_num = lambda word: word_num_map.get(word, words_size)
if txt_file!= None:
txt_label = get_ch_lable(txt_file)
labels_vector = list(map(to_num, txt_label))
return labels_vector
training_data =get_ch_lable(training_file)
print("Loaded training data...")
print(len(training_data))
counter = Counter(training_data)
words = sorted(counter)
words_size= len(words)
word_num_map = dict(zip(words, range(words_size)))
print('字表大小:', words_size)
wordlabel = get_ch_lable_v(training_file,word_num_map)
#参数设置
learning_rate = 0.001
training_iters = 10000
display_step = 1000
n_input = 4
n_hidden1 = 256
n_hidden2 = 512
n_hidden3 = 512
# tf Graph input
x = tf.placeholder("float", [None, n_input,1])
wordy = tf.placeholder("float", [None, words_size])
x1 = tf.reshape(x, [-1, n_input])
x2 = tf.split(x1,n_input,1)
# 2-layer LSTM, each layer has n_hidden units.
rnn_cell = rnn.MultiRNNCell([rnn.LSTMCell(n_hidden1),rnn.LSTMCell(n_hidden2),rnn.LSTMCell(n_hidden3)])
# generate prediction
outputs, states = rnn.static_rnn(rnn_cell, x2, dtype=tf.float32)
# last output
pred = tf.contrib.layers.fully_connected(outputs[-1],words_size,activation_fn = None)
# Loss optimizer
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=wordy))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)
# Model evaluation
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(wordy,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
savedir = "log/rnnword/"
saver = tf.train.Saver(max_to_keep=1) # 生成saver
# 启动session
with tf.Session() as session:
session.run(tf.global_variables_initializer())
step = 0
offset = random.randint(0,n_input+1)
end_offset = n_input + 1
acc_total = 0
loss_total = 0
kpt = tf.train.latest_checkpoint(savedir)
print("kpt:",kpt)
startepo= 0
if kpt!=None:
saver.restore(session, kpt)
ind = kpt.find("-")
startepo = int(kpt[ind+1:])
print(startepo)
step = startepo
while step < training_iters:
# 随机取一个位置偏移
if offset > (len(training_data)-end_offset):
offset = random.randint(0, n_input+1)
inwords = [ [wordlabel[ i]] for i in range(offset, offset+n_input) ]#按照指定的位置偏移获取后面的4个文字向量当作输入
inwords = np.reshape(np.array(inwords), [-1, n_input, 1])
out_onehot= np.zeros([words_size], dtype=float)
out_onehot[wordlabel[offset+n_input]] = 1.0
out_onehot = np.reshape(out_onehot,[1,-1])#所有的字都变成onehot
_, acc, lossval, onehot_pred = session.run([optimizer, accuracy, loss, pred],feed_dict={x: inwords, wordy: out_onehot})
loss_total += lossval
acc_total += acc
if (step+1) % display_step == 0:
print("Iter= " + str(step+1) + ", Average Loss= " + \
"{:.6f}".format(loss_total/display_step) + ", Average Accuracy= " + \
"{:.2f}%".format(100*acc_total/display_step))
acc_total = 0
loss_total = 0
in2 = [words [wordlabel[i]] for i in range(offset, offset + n_input)]
out2 = words [wordlabel[offset + n_input]]
out_pred=words[int(tf.argmax(onehot_pred, 1).eval())]
print("%s - [%s] vs [%s]" % (in2,out2,out_pred))
saver.save(session, savedir+"rnnwordtest.cpkt", global_step=step)
step += 1
offset += (n_input+1)#中间隔了一个，作为预测
print("Finished!")
saver.save(session, savedir+"rnnwordtest.cpkt", global_step=step)
print("Elapsed time: ", elapsed(time.time() - start_time))
while True:
prompt = "请输入%s个字: " % n_input
sentence = input(prompt)
inputword = sentence.strip()
if len(inputword) != n_input:
print("您输入的字符长度为：",len(inputword),"请输入4个字")
continue
try:
inputword = get_ch_lable_v(None,word_num_map,inputword)
for i in range(32):
keys = np.reshape(np.array(inputword), [-1, n_input, 1])
onehot_pred = session.run(pred, feed_dict={x: keys})
onehot_pred_index = int(tf.argmax(onehot_pred, 1).eval())
sentence = "%s%s" % (sentence,words[onehot_pred_index])
inputword = inputword[1:]
inputword.append(onehot_pred_index)
print(sentence)
except:
print("该字我还没学会")
实例70：word2vect
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
import random
import collections
from collections import Counter
import jieba
from sklearn.manifold import TSNE
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rcParams['font.sans-serif']=['SimHei']#用来正常显示中文标签
mpl.rcParams['font.family'] = 'STSong'
mpl.rcParams['font.size'] = 20
training_file = '人体阴阳与电能.txt'
#中文字
def get_ch_lable(txt_file):
labels= ""
with open(txt_file, 'rb') as f:
for label in f:
#labels =label.decode('utf-8')
labels =labels+label.decode('gb2312')
return labels
#分词
def fenci(training_data):
seg_list = jieba.cut(training_data) # 默认是精确模式
training_ci = " ".join(seg_list)
training_ci = training_ci.split()
#以空格将字符串分开
training_ci = np.array(training_ci)
training_ci = np.reshape(training_ci, [-1, ])
return training_ci
def build_dataset(words, n_words):
"""Process raw inputs into a dataset."""
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(n_words - 1))
dictionary = dict()
for word, _ in count:
dictionary[word] = len(dictionary)
data = list()
unk_count = 0
for word in words:
if word in dictionary:
index = dictionary[word]
else:
index = 0 # dictionary['UNK']
unk_count += 1
data.append(index)
count[0][1] = unk_count
reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
return data, count, dictionary, reversed_dictionary
training_data =get_ch_lable(training_file)
print("总字数",len(training_data))
training_ci =fenci(training_data)
#print(training_ci)
print("总词数",len(training_ci))
training_label, count, dictionary, words = build_dataset(training_ci, 350)
words_size = len(dictionary)
print("字典词数",words_size)
#print(training_label)#将文本转为词向量
#print(words)#每个编号对应的词
#print(dictionary)#每个词对应的编号
#print(count)#每个词对应的个数
####################################################
print('Sample data', training_label[:10], [words[i] for i in training_label[:10]])
data_index = 0
def generate_batch(data,batch_size, num_skips, skip_window):
global data_index
assert batch_size % num_skips == 0
assert num_skips <= 2 * skip_window
batch = np.ndarray(shape=(batch_size), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
span = 2 * skip_window + 1 # [ skip_window target skip_window ]
buffer = collections.deque(maxlen=span)
if data_index + span > len(data):
data_index = 0
buffer.extend(data[data_index:data_index + span])
data_index += span
for i in range(batch_size // num_skips):
target = skip_window # target label at the center of the buffer
targets_to_avoid = [skip_window]
for j in range(num_skips):
while target in targets_to_avoid:
target = random.randint(0, span - 1)
targets_to_avoid.append(target)
batch[i * num_skips + j] = buffer[skip_window]
labels[i * num_skips + j, 0] = buffer[target]
if data_index == len(data):
#print(data_index,len(data),span,len(data[:span]))
#buffer[:] = data[:span]
buffer = data[:span]
data_index = span
else:
buffer.append(data[data_index])
data_index += 1
# Backtrack a little bit to avoid skipping words in the end of a batch
data_index = (data_index + len(data) - span) % len(data)
return batch, labels
batch, labels = generate_batch(training_label,batch_size=8, num_skips=2, skip_window=1)
for i in range(8):# 取第一个字，后一个是标签，再取其前一个字当标签，
print(batch[i], words[batch[i]], '->', labels[i, 0], words[labels[i, 0]])
batch_size = 128
embedding_size = 128 # Dimension of the embedding vector.
skip_window = 1 # How many words to consider left and right.
num_skips = 2 # How many times to reuse an input to generate a label.
valid_size = 16 # Random set of words to evaluate similarity on.
valid_window =np.int32( words_size/2 ) # Only pick dev samples in the head of the distribution.
print("valid_window",valid_window)
valid_examples = np.random.choice(valid_window, valid_size, replace=False)#0-words_size/2,中的数取16个。不能重复。
num_sampled = 64 # Number of negative examples to sample.
tf.reset_default_graph()
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):
# Look up embeddings for inputs.
embeddings = tf.Variable(tf.random_uniform([words_size, embedding_size], -1.0, 1.0))#94个，每个128个向量
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
# Construct the variables for the NCE loss
nce_weights = tf.Variable( tf.truncated_normal([words_size, embedding_size],
stddev=1.0 / tf.sqrt(np.float32(embedding_size))))
nce_biases = tf.Variable(tf.zeros([words_size]))
# Compute the average NCE loss for the batch.
# tf.nce_loss automatically draws a new sample of the negative labels each
# time we evaluate the loss.
loss = tf.reduce_mean(
tf.nn.nce_loss(weights=nce_weights, biases=nce_biases,
labels=train_labels, inputs=embed,
num_sampled=num_sampled, num_classes=words_size))
# Construct the SGD optimizer using a learning rate of 1.0.
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
# Compute the cosine similarity between minibatch examples and all embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
print("________________________",similarity.shape)
#Begin training.
num_steps = 100001
with tf.Session() as sess:
sess.run( tf.global_variables_initializer() )
print('Initialized')
average_loss = 0
for step in range(num_steps):
batch_inputs, batch_labels = generate_batch(training_label, batch_size, num_skips, skip_window)
feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
# We perform one update step by evaluating the optimizer op (including it
# in the list of returned values for session.run()
_, loss_val = sess.run([optimizer, loss], feed_dict=feed_dict)
average_loss += loss_val
#通过打印测试可以看到 embed的值在逐渐的被调节
# emv = sess.run(embed,feed_dict = {train_inputs: [37,18]})
# print("emv-------------------",emv[0])
if step % 2000 == 0:
if step > 0:
average_loss /= 2000
# The average loss is an estimate of the loss over the last 2000 batches.
print('Average loss at step ', step, ': ', average_loss)
average_loss = 0
# Note that this is expensive (~20% slowdown if computed every 500 steps)
if step % 10000 == 0:
sim = similarity.eval(session=sess)
#print(valid_size)
for i in range(valid_size):
valid_word = words[valid_examples[i]]
#print("valid_word",valid_word)#16
top_k = 8 # number of nearest neighbors
nearest = (-sim[i, :]).argsort()[1:top_k + 1] #argsort函数返回的是数组值从小到大的索引值
#print("nearest",nearest,top_k)
log_str = 'Nearest to %s:' % valid_word
for k in range(top_k):
close_word = words[nearest[k]]
log_str = '%s,%s' % (log_str, close_word)
print(log_str)
final_embeddings = normalized_embeddings.eval()
def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
plt.figure(figsize=(18, 18)) # in inches
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
plt.scatter(x, y)
plt.annotate(label,xy=(x, y),xytext=(5, 2), textcoords='offset points',
ha='right',va='bottom')
plt.savefig(filename)
try:
# pylint: disable=g-import-not-at-top
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
plot_only = 80#输出100个词
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [words[i] for i in range(plot_only)]
#print(labels)
plot_with_labels(low_dim_embs, labels)
except ImportError:
print('Please install sklearn, matplotlib, and scipy to show embeddings.')
实例71：word2vect自定义候选采样
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
import random
import collections
from collections import Counter
import jieba
from sklearn.manifold import TSNE
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rcParams['font.sans-serif']=['SimHei']#用来正常显示中文标签
mpl.rcParams['font.family'] = 'STSong'
mpl.rcParams['font.size'] = 20
training_file = '人体阴阳与电能.txt'
#中文字
def get_ch_lable(txt_file):
labels= ""
with open(txt_file, 'rb') as f:
for label in f:
#labels =label.decode('utf-8')
labels =labels+label.decode('gb2312')
return labels
#分词
def fenci(training_data):
seg_list = jieba.cut(training_data) # 默认是精确模式
training_ci = " ".join(seg_list)
training_ci = training_ci.split()
#以空格将字符串分开
training_ci = np.array(training_ci)
training_ci = np.reshape(training_ci, [-1, ])
return training_ci
def build_dataset(words, n_words):
"""Process raw inputs into a dataset."""
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(n_words - 1))
dictionary = dict()
vocab_freqs = []
for word, nvocab in count:
dictionary[word] = len(dictionary)
vocab_freqs.append(nvocab)
data = list()
unk_count = 0
for word in words:
if word in dictionary:
index = dictionary[word]
else:
index = 0 # dictionary['UNK']
unk_count += 1
data.append(index)
count[0][1] = unk_count
reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
return data, count, dictionary, reversed_dictionary,vocab_freqs
training_data =get_ch_lable(training_file)
print("总字数",len(training_data))
training_ci =fenci(training_data)
#print(training_ci)
print("总词数",len(training_ci))
training_label, count, dictionary, words,vocab_freqs = build_dataset(training_ci, 350)
words_size = len(dictionary)
print("字典词数",words_size)
#print(training_label)#将文本转为词向量
#print(words)#每个编号对应的词
#print(dictionary)#每个词对应的编号
#print(count)#每个词对应的个数
####################################################
print('Sample data', training_label[:10], [words[i] for i in training_label[:10]])
data_index = 0
def generate_batch(data,batch_size, num_skips, skip_window):
global data_index
assert batch_size % num_skips == 0
assert num_skips <= 2 * skip_window
batch = np.ndarray(shape=(batch_size), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
span = 2 * skip_window + 1 # [ skip_window target skip_window ]
buffer = collections.deque(maxlen=span)
if data_index + span > len(data):
data_index = 0
buffer.extend(data[data_index:data_index + span])
data_index += span
for i in range(batch_size // num_skips):
target = skip_window # target label at the center of the buffer
targets_to_avoid = [skip_window]
for j in range(num_skips):
while target in targets_to_avoid:
target = random.randint(0, span - 1)
targets_to_avoid.append(target)
batch[i * num_skips + j] = buffer[skip_window]
labels[i * num_skips + j, 0] = buffer[target]
if data_index == len(data):
#print(data_index,len(data),span,len(data[:span]))
#buffer[:] = data[:span]
buffer = data[:span]
data_index = span
else:
buffer.append(data[data_index])
data_index += 1
# Backtrack a little bit to avoid skipping words in the end of a batch
data_index = (data_index + len(data) - span) % len(data)
return batch, labels
batch, labels = generate_batch(training_label,batch_size=8, num_skips=2, skip_window=1)
for i in range(8):# 取第一个字，后一个是标签，再取其前一个字当标签，
print(batch[i], words[batch[i]], '->', labels[i, 0], words[labels[i, 0]])
batch_size = 128
embedding_size = 128 # Dimension of the embedding vector.
skip_window = 1 # How many words to consider left and right.
num_skips = 2 # How many times to reuse an input to generate a label.
valid_size = 16 # Random set of words to evaluate similarity on.
valid_window =np.int32( words_size/2 ) # Only pick dev samples in the head of the distribution.
print("valid_window",valid_window)
valid_examples = np.random.choice(valid_window, valid_size, replace=False)#0-words_size/2,中的数取16个。不能重复。
num_sampled = 64 # Number of negative examples to sample.
tf.reset_default_graph()
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):
# Look up embeddings for inputs.
embeddings = tf.Variable(tf.random_uniform([words_size, embedding_size], -1.0, 1.0))#94个，每个128个向量
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
# Construct the variables for the NCE loss
nce_weights = tf.Variable( tf.truncated_normal([words_size, embedding_size],
stddev=1.0 / tf.sqrt(np.float32(embedding_size))))
nce_biases = tf.Variable(tf.zeros([words_size]))
vocab_freqs[0] = 90
sampled = tf.nn.fixed_unigram_candidate_sampler(
true_classes=tf.cast(train_labels,tf.int64),
num_true=1,
num_sampled=num_sampled,
unique=True,
range_max=words_size,
unigrams=vocab_freqs)
loss = tf.reduce_mean(
tf.nn.sampled_softmax_loss(weights=nce_weights, biases=nce_biases,
labels=train_labels, inputs=embed,
num_sampled=num_sampled, num_classes=words_size,sampled_values=sampled))
# Construct the SGD optimizer using a learning rate of 1.0.
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
# Compute the cosine similarity between minibatch examples and all embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
print("________________________",similarity.shape)
#Begin training.
num_steps = 100001
with tf.Session() as sess:
sess.run( tf.global_variables_initializer() )
print('Initialized')
average_loss = 0
for step in range(num_steps):
batch_inputs, batch_labels = generate_batch(training_label, batch_size, num_skips, skip_window)
feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
# We perform one update step by evaluating the optimizer op (including it
# in the list of returned values for session.run()
_, loss_val = sess.run([optimizer, loss], feed_dict=feed_dict)
average_loss += loss_val
#通过打印测试可以看到 embed的值在逐渐的被调节
# emv = sess.run(embed,feed_dict = {train_inputs: [37,18]})
# print("emv-------------------",emv[0])
if step % 2000 == 0:
if step > 0:
average_loss /= 2000
# The average loss is an estimate of the loss over the last 2000 batches.
print('Average loss at step ', step, ': ', average_loss)
average_loss = 0
# Note that this is expensive (~20% slowdown if computed every 500 steps)
if step % 10000 == 0:
sim = similarity.eval(session=sess)
#print(valid_size)
for i in range(valid_size):
valid_word = words[valid_examples[i]]
#print("valid_word",valid_word)#16
top_k = 8 # number of nearest neighbors
nearest = (-sim[i, :]).argsort()[1:top_k + 1] #argsort函数返回的是数组值从小到大的索引值
#print("nearest",nearest,top_k)
log_str = 'Nearest to %s:' % valid_word
for k in range(top_k):
close_word = words[nearest[k]]
log_str = '%s,%s' % (log_str, close_word)
print(log_str)
final_embeddings = normalized_embeddings.eval()
def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
plt.figure(figsize=(18, 18)) # in inches
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
plt.scatter(x, y)
plt.annotate(label,xy=(x, y),xytext=(5, 2), textcoords='offset points',
ha='right',va='bottom')
plt.savefig(filename)
try:
# pylint: disable=g-import-not-at-top
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
plot_only = 80#输出100个词
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [words[i] for i in range(plot_only)]
#print(labels)
plot_with_labels(low_dim_embs, labels)
except ImportError:
print('Please install sklearn, matplotlib, and scipy to show embeddings.')
185
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
