实例55:RNN
# -*- coding: utf-8 -*-
# 1. 定义基本函数
import copy, numpy as np
np.random.seed(0) #随机数生成器的种子,可以每次得到一样的值
# compute sigmoid nonlinearity
def sigmoid(x): #激活函数
output = 1/(1+np.exp(-x))
return output
# convert output of sigmoid function to its derivative
def sigmoid_output_to_derivative(output):#激活函数的导数
return output*(1-output)
# 2. 建立二进制映射
int2binary = {} #整数到其二进制表示的映射
binary_dim = 8 #暂时制作256以内的减法
## 计算0-256的二进制表示
largest_number = pow(2,binary_dim)
binary = np.unpackbits(
np.array([range(largest_number)],dtype=np.uint8).T,axis=1)
for i in range(largest_number):
int2binary[i] = binary[i]
# 3. 定义参数
# input variables
alpha = 0.9 #学习速率
input_dim = 2 #输入的维度是2
hidden_dim = 16
output_dim = 1 #输出维度为1
# initialize neural network weights
synapse_0 = (2*np.random.random((input_dim,hidden_dim)) - 1)*0.05 #维度为2*16, 2是输入维度,16是隐藏层维度
synapse_1 = (2*np.random.random((hidden_dim,output_dim)) - 1)*0.05
synapse_h = (2*np.random.random((hidden_dim,hidden_dim)) - 1)*0.05
# => [-0.05, 0.05),
# 用于存放反向传播的权重更新值
synapse_0_update = np.zeros_like(synapse_0)
synapse_1_update = np.zeros_like(synapse_1)
synapse_h_update = np.zeros_like(synapse_h)
# 4. 准备样本数据
# training
for j in range(10000):
#生成一个数字a
a_int = np.random.randint(largest_number)
#生成一个数字b,b的最大值取的是largest_number/2,作为被减数,让它小一点。
b_int = np.random.randint(largest_number/2)
#如果生成的b大了,那么交换一下
if a_int<b_int:
tt = a_int
b_int = a_int
a_int=tt
a = int2binary[a_int] # binary encoding
b = int2binary[b_int] # binary encoding
# true answer
c_int = a_int - b_int
c = int2binary[c_int]
# 5. 模型初始化
# 存储神经网络的预测值
d = np.zeros_like(c)
overallError = 0 #每次把总误差清零
layer_2_deltas = list() #存储每个时间点输出层的误差
layer_1_values = list() #存储每个时间点隐藏层的值
layer_1_values.append(np.ones(hidden_dim)*0.1) # 一开始没有隐藏层,所以初始化一下原始值为0.1
# 6. 正向传播
# moving along the positions in the binary encoding
for position in range(binary_dim):#循环遍历每一个二进制位
# generate input and output
X = np.array([[a[binary_dim - position - 1],b[binary_dim - position - 1]]])#从右到左,每次去两个输入数字的一个bit位
y = np.array([[c[binary_dim - position - 1]]]).T#正确答案
# hidden layer (input ~+ prev_hidden)
layer_1 = sigmoid(np.dot(X,synapse_0) + np.dot(layer_1_values[-1],synapse_h))#(输入层 + 之前的隐藏层) -> 新的隐藏层,这是体现循环神经网络的最核心的地方!!!
# output layer (new binary representation)
layer_2 = sigmoid(np.dot(layer_1,synapse_1)) #隐藏层 * 隐藏层到输出层的转化矩阵synapse_1 -> 输出层
layer_2_error = y - layer_2 #预测误差
layer_2_deltas.append((layer_2_error)*sigmoid_output_to_derivative(layer_2)) #把每一个时间点的误差导数都记录下来
overallError += np.abs(layer_2_error[0])#总误差
d[binary_dim - position - 1] = np.round(layer_2[0][0]) #记录下每一个预测bit位
# store hidden layer so we can use it in the next timestep
layer_1_values.append(copy.deepcopy(layer_1))#记录下隐藏层的值,在下一个时间点用
future_layer_1_delta = np.zeros(hidden_dim)
# 7. 反向传播
#反向传播,从最后一个时间点到第一个时间点
for position in range(binary_dim):
X = np.array([[a[position],b[position]]]) #最后一次的两个输入
layer_1 = layer_1_values[-position-1] #当前时间点的隐藏层
prev_layer_1 = layer_1_values[-position-2] #前一个时间点的隐藏层
# error at output layer
layer_2_delta = layer_2_deltas[-position-1] #当前时间点输出层导数
# error at hidden layer
# 通过后一个时间点(因为是反向传播)的隐藏层误差和当前时间点的输出层误差,计算当前时间点的隐藏层误差
layer_1_delta = (future_layer_1_delta.dot(synapse_h.T) + layer_2_delta.dot(synapse_1.T)) * sigmoid_output_to_derivative(layer_1)
# 等到完成了所有反向传播误差计算, 才会更新权重矩阵,先暂时把更新矩阵存起来。
synapse_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)
synapse_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)
synapse_0_update += X.T.dot(layer_1_delta)
future_layer_1_delta = layer_1_delta
# 完成所有反向传播之后,更新权重矩阵。并把矩阵变量清零
synapse_0 += synapse_0_update * alpha
synapse_1 += synapse_1_update * alpha
synapse_h += synapse_h_update * alpha
synapse_0_update *= 0
synapse_1_update *= 0
synapse_h_update *= 0
# 8. 输出结果
# print out progress
if(j % 800 == 0):
#print(synapse_0,synapse_h,synapse_1)
print("总误差:" + str(overallError))
print("Pred:" + str(d))
print("True:" + str(c))
out = 0
for index,x in enumerate(reversed(d)):
out += x*pow(2,index)
print(str(a_int) + " - " + str(b_int) + " = " + str(out))
print("------------")
实例56:使用RNN网络拟合回声信号序列
# -*- coding: utf-8 -*-
# 1. 定义参数生成样本数据
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
num_epochs = 5
total_series_length = 50000
truncated_backprop_length = 15
state_size = 4
num_classes = 2
echo_step = 3
batch_size = 5
num_batches = total_series_length//batch_size//truncated_backprop_length
def generateData():
x = np.array(np.random.choice(2, total_series_length, p=[0.5, 0.5]))#在0 和1 中选择total_series_length个数
y = np.roll(x, echo_step)#向右循环移位【1111000】---【0001111】
y[0:echo_step] = 0
x = x.reshape((batch_size, -1)) # 5,10000
y = y.reshape((batch_size, -1))
return (x, y)
# 2. 定义占位符处理输入数据
batchX_placeholder = tf.placeholder(tf.float32, [batch_size, truncated_backprop_length])
batchY_placeholder = tf.placeholder(tf.int32, [batch_size, truncated_backprop_length])
init_state = tf.placeholder(tf.float32, [batch_size, state_size])
# Unpack columns
inputs_series = tf.unstack(batchX_placeholder, axis=1)#truncated_backprop_length个序列
labels_series = tf.unstack(batchY_placeholder, axis=1)
# 3. 定义网络结构
current_state = init_state
predictions_series = []
losses =[]
for current_input, labels in zip(inputs_series,labels_series):
#for current_input in inputs_series:
current_input = tf.reshape(current_input, [batch_size, 1])
input_and_state_concatenated = tf.concat([current_input, current_state],1) # current_state 4 +1
next_state = tf.contrib.layers.fully_connected(input_and_state_concatenated,state_size
,activation_fn=tf.tanh)
current_state = next_state
logits =tf.contrib.layers.fully_connected(next_state,num_classes,activation_fn=None)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels,logits=logits)
losses.append(loss)
predictions = tf.nn.softmax(logits)
predictions_series.append(predictions)
total_loss = tf.reduce_mean(losses)
train_step = tf.train.AdagradOptimizer(0.3).minimize(total_loss)
# plot 函数
def plot(loss_list, predictions_series, batchX, batchY):
plt.subplot(2, 3, 1)
plt.cla()
plt.plot(loss_list)
for batch_series_idx in range(batch_size):
one_hot_output_series = np.array(predictions_series)[:, batch_series_idx, :]
single_output_series = np.array([(1 if out[0] < 0.5 else 0) for out in one_hot_output_series])
plt.subplot(2, 3, batch_series_idx + 2)
plt.cla()
plt.axis([0, truncated_backprop_length, 0, 2])
left_offset = range(truncated_backprop_length)
left_offset2 = range(echo_step,truncated_backprop_length+echo_step)
label1 = "past values"
label2 = "True echo values"
label3 = "Predictions"
plt.plot(left_offset2, batchX[batch_series_idx, :]*0.2+1.5, "o--b", label=label1)
plt.plot(left_offset, batchY[batch_series_idx, :]*0.2+0.8,"x--b", label=label2)
plt.plot(left_offset, single_output_series*0.2+0.1 , "o--y", label=label3)
plt.legend(loc='best')
plt.draw()
plt.pause(0.0001)
# 4. 建立session训练数据
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
plt.ion()
plt.figure()
plt.show()
loss_list = []
for epoch_idx in range(num_epochs):
x,y = generateData()
_current_state = np.zeros((batch_size, state_size))
print("New data, epoch", epoch_idx)
for batch_idx in range(num_batches):#50000/ 5 /15=分成多少段
start_idx = batch_idx * truncated_backprop_length
end_idx = start_idx + truncated_backprop_length
batchX = x[:,start_idx:end_idx]
batchY = y[:,start_idx:end_idx]
_total_loss, _train_step, _current_state, _predictions_series = sess.run(
[total_loss, train_step, current_state, predictions_series],
feed_dict={
batchX_placeholder:batchX,
batchY_placeholder:batchY,
init_state:_current_state
})
loss_list.append(_total_loss)
# 5. 测试模型及可视化
if batch_idx%100 == 0:
print("Step",batch_idx, "Loss", _total_loss)
plot(loss_list, _predictions_series, batchX, batchY)
plt.ioff()
plt.show()
实例57:构造LSTM对MNIST分类(BasicCell & LSTMCell)
# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
#outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.100226, Training Accuracy= 0.35938 Iter 2560, Minibatch Loss= 1.727162, Training Accuracy= 0.46875 Iter 3840, Minibatch Loss= 1.655166, Training Accuracy= 0.41406 Iter 5120, Minibatch Loss= 1.134178, Training Accuracy= 0.60938 Iter 6400, Minibatch Loss= 1.077124, Training Accuracy= 0.70312 Iter 7680, Minibatch Loss= 0.919439, Training Accuracy= 0.74219 Iter 8960, Minibatch Loss= 0.661384, Training Accuracy= 0.77344 Iter 10240, Minibatch Loss= 0.611414, Training Accuracy= 0.79688 Iter 11520, Minibatch Loss= 0.649199, Training Accuracy= 0.77344 Iter 12800, Minibatch Loss= 0.732224, Training Accuracy= 0.75781 Iter 14080, Minibatch Loss= 0.520225, Training Accuracy= 0.84375 Iter 15360, Minibatch Loss= 0.435234, Training Accuracy= 0.86719 Iter 16640, Minibatch Loss= 0.354433, Training Accuracy= 0.91406 Iter 17920, Minibatch Loss= 0.391198, Training Accuracy= 0.85938 Iter 19200, Minibatch Loss= 0.574429, Training Accuracy= 0.80469 Iter 20480, Minibatch Loss= 0.508154, Training Accuracy= 0.83594 Iter 21760, Minibatch Loss= 0.281986, Training Accuracy= 0.91406 Iter 23040, Minibatch Loss= 0.427695, Training Accuracy= 0.86719 Iter 24320, Minibatch Loss= 0.286359, Training Accuracy= 0.92969 Iter 25600, Minibatch Loss= 0.278457, Training Accuracy= 0.92188 Iter 26880, Minibatch Loss= 0.228263, Training Accuracy= 0.95312 Iter 28160, Minibatch Loss= 0.280953, Training Accuracy= 0.90625 Iter 29440, Minibatch Loss= 0.234286, Training Accuracy= 0.93750 Iter 30720, Minibatch Loss= 0.310759, Training Accuracy= 0.89062 Iter 32000, Minibatch Loss= 0.365036, Training Accuracy= 0.91406 Iter 33280, Minibatch Loss= 0.264047, Training Accuracy= 0.93750 Iter 34560, Minibatch Loss= 0.290281, Training Accuracy= 0.90625 Iter 35840, Minibatch Loss= 0.196424, Training Accuracy= 0.92969 Iter 37120, Minibatch Loss= 0.198498, Training Accuracy= 0.94531 Iter 38400, Minibatch Loss= 0.342719, Training Accuracy= 0.89844 Iter 39680, Minibatch Loss= 0.319853, Training Accuracy= 0.90625 Iter 40960, Minibatch Loss= 0.209966, Training Accuracy= 0.94531 Iter 42240, Minibatch Loss= 0.109240, Training Accuracy= 0.96875 Iter 43520, Minibatch Loss= 0.245075, Training Accuracy= 0.92188 Iter 44800, Minibatch Loss= 0.169283, Training Accuracy= 0.95312 Iter 46080, Minibatch Loss= 0.141722, Training Accuracy= 0.94531 Iter 47360, Minibatch Loss= 0.139368, Training Accuracy= 0.96875 Iter 48640, Minibatch Loss= 0.129791, Training Accuracy= 0.95312 Iter 49920, Minibatch Loss= 0.135869, Training Accuracy= 0.95312 Iter 51200, Minibatch Loss= 0.167792, Training Accuracy= 0.92969 Iter 52480, Minibatch Loss= 0.219980, Training Accuracy= 0.94531 Iter 53760, Minibatch Loss= 0.253420, Training Accuracy= 0.89844 Iter 55040, Minibatch Loss= 0.310215, Training Accuracy= 0.91406 Iter 56320, Minibatch Loss= 0.140659, Training Accuracy= 0.94531 Iter 57600, Minibatch Loss= 0.147263, Training Accuracy= 0.96094 Iter 58880, Minibatch Loss= 0.145595, Training Accuracy= 0.96875 Iter 60160, Minibatch Loss= 0.140386, Training Accuracy= 0.94531 Iter 61440, Minibatch Loss= 0.112856, Training Accuracy= 0.96094 Iter 62720, Minibatch Loss= 0.066107, Training Accuracy= 0.98438 Iter 64000, Minibatch Loss= 0.141188, Training Accuracy= 0.94531 Iter 65280, Minibatch Loss= 0.124689, Training Accuracy= 0.97656 Iter 66560, Minibatch Loss= 0.136360, Training Accuracy= 0.96094 Iter 67840, Minibatch Loss= 0.127406, Training Accuracy= 0.94531 Iter 69120, Minibatch Loss= 0.190868, Training Accuracy= 0.94531 Iter 70400, Minibatch Loss= 0.146545, Training Accuracy= 0.94531 Iter 71680, Minibatch Loss= 0.144414, Training Accuracy= 0.95312 Iter 72960, Minibatch Loss= 0.139988, Training Accuracy= 0.93750 Iter 74240, Minibatch Loss= 0.078669, Training Accuracy= 0.97656 Iter 75520, Minibatch Loss= 0.194032, Training Accuracy= 0.93750 Iter 76800, Minibatch Loss= 0.123782, Training Accuracy= 0.96875 Iter 78080, Minibatch Loss= 0.153625, Training Accuracy= 0.96094 Iter 79360, Minibatch Loss= 0.125683, Training Accuracy= 0.97656 Iter 80640, Minibatch Loss= 0.146972, Training Accuracy= 0.93750 Iter 81920, Minibatch Loss= 0.175487, Training Accuracy= 0.96094 Iter 83200, Minibatch Loss= 0.078552, Training Accuracy= 0.98438 Iter 84480, Minibatch Loss= 0.136101, Training Accuracy= 0.95312 Iter 85760, Minibatch Loss= 0.141502, Training Accuracy= 0.95312 Iter 87040, Minibatch Loss= 0.068094, Training Accuracy= 0.98438 Iter 88320, Minibatch Loss= 0.289425, Training Accuracy= 0.89844 Iter 89600, Minibatch Loss= 0.127497, Training Accuracy= 0.95312 Iter 90880, Minibatch Loss= 0.090173, Training Accuracy= 0.97656 Iter 92160, Minibatch Loss= 0.135557, Training Accuracy= 0.95312 Iter 93440, Minibatch Loss= 0.122439, Training Accuracy= 0.96094 Iter 94720, Minibatch Loss= 0.171625, Training Accuracy= 0.94531 Iter 96000, Minibatch Loss= 0.140195, Training Accuracy= 0.94531 Iter 97280, Minibatch Loss= 0.156978, Training Accuracy= 0.96094 Iter 98560, Minibatch Loss= 0.050950, Training Accuracy= 0.99219 Iter 99840, Minibatch Loss= 0.086917, Training Accuracy= 0.97656 Finished! Testing Accuracy: 0.9921875
# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
#outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.132512, Training Accuracy= 0.30469 Iter 2560, Minibatch Loss= 2.001817, Training Accuracy= 0.28125 Iter 3840, Minibatch Loss= 1.523742, Training Accuracy= 0.51562 Iter 5120, Minibatch Loss= 1.272301, Training Accuracy= 0.64844 Iter 6400, Minibatch Loss= 1.060405, Training Accuracy= 0.65625 Iter 7680, Minibatch Loss= 0.889486, Training Accuracy= 0.69531 Iter 8960, Minibatch Loss= 0.854698, Training Accuracy= 0.69531 Iter 10240, Minibatch Loss= 0.786288, Training Accuracy= 0.77344 Iter 11520, Minibatch Loss= 0.758092, Training Accuracy= 0.77344 Iter 12800, Minibatch Loss= 0.784820, Training Accuracy= 0.77344 Iter 14080, Minibatch Loss= 0.383889, Training Accuracy= 0.91406 Iter 15360, Minibatch Loss= 0.458686, Training Accuracy= 0.82031 Iter 16640, Minibatch Loss= 0.446535, Training Accuracy= 0.85156 Iter 17920, Minibatch Loss= 0.593214, Training Accuracy= 0.83594 Iter 19200, Minibatch Loss= 0.483660, Training Accuracy= 0.85156 Iter 20480, Minibatch Loss= 0.413878, Training Accuracy= 0.87500 Iter 21760, Minibatch Loss= 0.438487, Training Accuracy= 0.85938 Iter 23040, Minibatch Loss= 0.440690, Training Accuracy= 0.89844 Iter 24320, Minibatch Loss= 0.395759, Training Accuracy= 0.88281 Iter 25600, Minibatch Loss= 0.303816, Training Accuracy= 0.91406 Iter 26880, Minibatch Loss= 0.457287, Training Accuracy= 0.85156 Iter 28160, Minibatch Loss= 0.435749, Training Accuracy= 0.89062 Iter 29440, Minibatch Loss= 0.282319, Training Accuracy= 0.92188 Iter 30720, Minibatch Loss= 0.341908, Training Accuracy= 0.90625 Iter 32000, Minibatch Loss= 0.413945, Training Accuracy= 0.84375 Iter 33280, Minibatch Loss= 0.277253, Training Accuracy= 0.90625 Iter 34560, Minibatch Loss= 0.265192, Training Accuracy= 0.92188 Iter 35840, Minibatch Loss= 0.273202, Training Accuracy= 0.89844 Iter 37120, Minibatch Loss= 0.188953, Training Accuracy= 0.95312 Iter 38400, Minibatch Loss= 0.401897, Training Accuracy= 0.89844 Iter 39680, Minibatch Loss= 0.277849, Training Accuracy= 0.91406 Iter 40960, Minibatch Loss= 0.256338, Training Accuracy= 0.91406 Iter 42240, Minibatch Loss= 0.259410, Training Accuracy= 0.92188 Iter 43520, Minibatch Loss= 0.286601, Training Accuracy= 0.95312 Iter 44800, Minibatch Loss= 0.163870, Training Accuracy= 0.96094 Iter 46080, Minibatch Loss= 0.161053, Training Accuracy= 0.96094 Iter 47360, Minibatch Loss= 0.250546, Training Accuracy= 0.92969 Iter 48640, Minibatch Loss= 0.134387, Training Accuracy= 0.96094 Iter 49920, Minibatch Loss= 0.173047, Training Accuracy= 0.92969 Iter 51200, Minibatch Loss= 0.337741, Training Accuracy= 0.90625 Iter 52480, Minibatch Loss= 0.207449, Training Accuracy= 0.93750 Iter 53760, Minibatch Loss= 0.156933, Training Accuracy= 0.94531 Iter 55040, Minibatch Loss= 0.199040, Training Accuracy= 0.94531 Iter 56320, Minibatch Loss= 0.147556, Training Accuracy= 0.95312 Iter 57600, Minibatch Loss= 0.148920, Training Accuracy= 0.94531 Iter 58880, Minibatch Loss= 0.144406, Training Accuracy= 0.94531 Iter 60160, Minibatch Loss= 0.114759, Training Accuracy= 0.96875 Iter 61440, Minibatch Loss= 0.145373, Training Accuracy= 0.95312 Iter 62720, Minibatch Loss= 0.154870, Training Accuracy= 0.95312 Iter 64000, Minibatch Loss= 0.176493, Training Accuracy= 0.95312 Iter 65280, Minibatch Loss= 0.124761, Training Accuracy= 0.96094 Iter 66560, Minibatch Loss= 0.060221, Training Accuracy= 0.98438 Iter 67840, Minibatch Loss= 0.154883, Training Accuracy= 0.95312 Iter 69120, Minibatch Loss= 0.141949, Training Accuracy= 0.95312 Iter 70400, Minibatch Loss= 0.289304, Training Accuracy= 0.91406 Iter 71680, Minibatch Loss= 0.167781, Training Accuracy= 0.94531 Iter 72960, Minibatch Loss= 0.239404, Training Accuracy= 0.93750 Iter 74240, Minibatch Loss= 0.138790, Training Accuracy= 0.96094 Iter 75520, Minibatch Loss= 0.176216, Training Accuracy= 0.92969 Iter 76800, Minibatch Loss= 0.179790, Training Accuracy= 0.95312 Iter 78080, Minibatch Loss= 0.157953, Training Accuracy= 0.93750 Iter 79360, Minibatch Loss= 0.213766, Training Accuracy= 0.92188 Iter 80640, Minibatch Loss= 0.080645, Training Accuracy= 0.97656 Iter 81920, Minibatch Loss= 0.191703, Training Accuracy= 0.95312 Iter 83200, Minibatch Loss= 0.172994, Training Accuracy= 0.96094 Iter 84480, Minibatch Loss= 0.210067, Training Accuracy= 0.93750 Iter 85760, Minibatch Loss= 0.165411, Training Accuracy= 0.94531 Iter 87040, Minibatch Loss= 0.087349, Training Accuracy= 0.97656 Iter 88320, Minibatch Loss= 0.189240, Training Accuracy= 0.93750 Iter 89600, Minibatch Loss= 0.153909, Training Accuracy= 0.94531 Iter 90880, Minibatch Loss= 0.231399, Training Accuracy= 0.92188 Iter 92160, Minibatch Loss= 0.132192, Training Accuracy= 0.95312 Iter 93440, Minibatch Loss= 0.078557, Training Accuracy= 0.98438 Iter 94720, Minibatch Loss= 0.081286, Training Accuracy= 0.98438 Iter 96000, Minibatch Loss= 0.154948, Training Accuracy= 0.96094 Iter 97280, Minibatch Loss= 0.132285, Training Accuracy= 0.95312 Iter 98560, Minibatch Loss= 0.107341, Training Accuracy= 0.97656 Iter 99840, Minibatch Loss= 0.068848, Training Accuracy= 1.00000 Finished! Testing Accuracy: 0.984375
实例58:构造单层GRU网络对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
gru = tf.contrib.rnn.GRUCell(n_hidden)
outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
#outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.110982, Training Accuracy= 0.35156 Iter 2560, Minibatch Loss= 1.813759, Training Accuracy= 0.36719 Iter 3840, Minibatch Loss= 1.430715, Training Accuracy= 0.57812 Iter 5120, Minibatch Loss= 1.243752, Training Accuracy= 0.53906 Iter 6400, Minibatch Loss= 1.036775, Training Accuracy= 0.66406 Iter 7680, Minibatch Loss= 0.958580, Training Accuracy= 0.65625 Iter 8960, Minibatch Loss= 0.797127, Training Accuracy= 0.75000 Iter 10240, Minibatch Loss= 0.834968, Training Accuracy= 0.71094 Iter 11520, Minibatch Loss= 0.762974, Training Accuracy= 0.72656 Iter 12800, Minibatch Loss= 0.702314, Training Accuracy= 0.78125 Iter 14080, Minibatch Loss= 0.631187, Training Accuracy= 0.75000 Iter 15360, Minibatch Loss= 0.601635, Training Accuracy= 0.82031 Iter 16640, Minibatch Loss= 0.539469, Training Accuracy= 0.84375 Iter 17920, Minibatch Loss= 0.446828, Training Accuracy= 0.89844 Iter 19200, Minibatch Loss= 0.491736, Training Accuracy= 0.86719 Iter 20480, Minibatch Loss= 0.443790, Training Accuracy= 0.85156 Iter 21760, Minibatch Loss= 0.360417, Training Accuracy= 0.90625 Iter 23040, Minibatch Loss= 0.399639, Training Accuracy= 0.89844 Iter 24320, Minibatch Loss= 0.365533, Training Accuracy= 0.90625 Iter 25600, Minibatch Loss= 0.361865, Training Accuracy= 0.89062 Iter 26880, Minibatch Loss= 0.314257, Training Accuracy= 0.89844 Iter 28160, Minibatch Loss= 0.323813, Training Accuracy= 0.88281 Iter 29440, Minibatch Loss= 0.258019, Training Accuracy= 0.92969 Iter 30720, Minibatch Loss= 0.362180, Training Accuracy= 0.90625 Iter 32000, Minibatch Loss= 0.300678, Training Accuracy= 0.90625 Iter 33280, Minibatch Loss= 0.345444, Training Accuracy= 0.88281 Iter 34560, Minibatch Loss= 0.315548, Training Accuracy= 0.90625 Iter 35840, Minibatch Loss= 0.149648, Training Accuracy= 0.94531 Iter 37120, Minibatch Loss= 0.208241, Training Accuracy= 0.95312 Iter 38400, Minibatch Loss= 0.240656, Training Accuracy= 0.92188 Iter 39680, Minibatch Loss= 0.242481, Training Accuracy= 0.94531 Iter 40960, Minibatch Loss= 0.198655, Training Accuracy= 0.94531 Iter 42240, Minibatch Loss= 0.169093, Training Accuracy= 0.96094 Iter 43520, Minibatch Loss= 0.144338, Training Accuracy= 0.96094 Iter 44800, Minibatch Loss= 0.120352, Training Accuracy= 0.96875 Iter 46080, Minibatch Loss= 0.260023, Training Accuracy= 0.92969 Iter 47360, Minibatch Loss= 0.259179, Training Accuracy= 0.92188 Iter 48640, Minibatch Loss= 0.169763, Training Accuracy= 0.95312 Iter 49920, Minibatch Loss= 0.295729, Training Accuracy= 0.87500 Iter 51200, Minibatch Loss= 0.262093, Training Accuracy= 0.94531 Iter 52480, Minibatch Loss= 0.160087, Training Accuracy= 0.94531 Iter 53760, Minibatch Loss= 0.216548, Training Accuracy= 0.92969 Iter 55040, Minibatch Loss= 0.197999, Training Accuracy= 0.92969 Iter 56320, Minibatch Loss= 0.209844, Training Accuracy= 0.93750 Iter 57600, Minibatch Loss= 0.145806, Training Accuracy= 0.96094 Iter 58880, Minibatch Loss= 0.145609, Training Accuracy= 0.92969 Iter 60160, Minibatch Loss= 0.141855, Training Accuracy= 0.96875 Iter 61440, Minibatch Loss= 0.149774, Training Accuracy= 0.96094 Iter 62720, Minibatch Loss= 0.151638, Training Accuracy= 0.95312 Iter 64000, Minibatch Loss= 0.183334, Training Accuracy= 0.95312 Iter 65280, Minibatch Loss= 0.135904, Training Accuracy= 0.96094 Iter 66560, Minibatch Loss= 0.099675, Training Accuracy= 0.97656 Iter 67840, Minibatch Loss= 0.122692, Training Accuracy= 0.96094 Iter 69120, Minibatch Loss= 0.146226, Training Accuracy= 0.94531 Iter 70400, Minibatch Loss= 0.180213, Training Accuracy= 0.96094 Iter 71680, Minibatch Loss= 0.095388, Training Accuracy= 0.96875 Iter 72960, Minibatch Loss= 0.151607, Training Accuracy= 0.95312 Iter 74240, Minibatch Loss= 0.138024, Training Accuracy= 0.95312 Iter 75520, Minibatch Loss= 0.077417, Training Accuracy= 0.98438 Iter 76800, Minibatch Loss= 0.142044, Training Accuracy= 0.96094 Iter 78080, Minibatch Loss= 0.096436, Training Accuracy= 0.96875 Iter 79360, Minibatch Loss= 0.170768, Training Accuracy= 0.94531 Iter 80640, Minibatch Loss= 0.244333, Training Accuracy= 0.94531 Iter 81920, Minibatch Loss= 0.049825, Training Accuracy= 0.98438 Iter 83200, Minibatch Loss= 0.100651, Training Accuracy= 0.96094 Iter 84480, Minibatch Loss= 0.100706, Training Accuracy= 0.96875 Iter 85760, Minibatch Loss= 0.125473, Training Accuracy= 0.96094 Iter 87040, Minibatch Loss= 0.049680, Training Accuracy= 0.99219 Iter 88320, Minibatch Loss= 0.077539, Training Accuracy= 0.97656 Iter 89600, Minibatch Loss= 0.124252, Training Accuracy= 0.96094 Iter 90880, Minibatch Loss= 0.039592, Training Accuracy= 0.99219 Iter 92160, Minibatch Loss= 0.108718, Training Accuracy= 0.96875 Iter 93440, Minibatch Loss= 0.085434, Training Accuracy= 0.97656 Iter 94720, Minibatch Loss= 0.164223, Training Accuracy= 0.95312 Iter 96000, Minibatch Loss= 0.174773, Training Accuracy= 0.92969 Iter 97280, Minibatch Loss= 0.062409, Training Accuracy= 0.98438 Iter 98560, Minibatch Loss= 0.065892, Training Accuracy= 0.98438 Iter 99840, Minibatch Loss= 0.109494, Training Accuracy= 0.96875 Finished! Testing Accuracy: 0.96875
实例59:创建动态单层RNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.117727, Training Accuracy= 0.28906 Iter 2560, Minibatch Loss= 1.873549, Training Accuracy= 0.37500 Iter 3840, Minibatch Loss= 1.643477, Training Accuracy= 0.46875 Iter 5120, Minibatch Loss= 1.384699, Training Accuracy= 0.54688 Iter 6400, Minibatch Loss= 1.180172, Training Accuracy= 0.60938 Iter 7680, Minibatch Loss= 1.117720, Training Accuracy= 0.59375 Iter 8960, Minibatch Loss= 0.758485, Training Accuracy= 0.72656 Iter 10240, Minibatch Loss= 0.766036, Training Accuracy= 0.76562 Iter 11520, Minibatch Loss= 0.644146, Training Accuracy= 0.80469 Iter 12800, Minibatch Loss= 0.611225, Training Accuracy= 0.82812 Iter 14080, Minibatch Loss= 0.442834, Training Accuracy= 0.84375 Iter 15360, Minibatch Loss= 0.449818, Training Accuracy= 0.88281 Iter 16640, Minibatch Loss= 0.438875, Training Accuracy= 0.87500 Iter 17920, Minibatch Loss= 0.267779, Training Accuracy= 0.91406 Iter 19200, Minibatch Loss= 0.243440, Training Accuracy= 0.95312 Iter 20480, Minibatch Loss= 0.444132, Training Accuracy= 0.85938 Iter 21760, Minibatch Loss= 0.346954, Training Accuracy= 0.90625 Iter 23040, Minibatch Loss= 0.175107, Training Accuracy= 0.95312 Iter 24320, Minibatch Loss= 0.268770, Training Accuracy= 0.92969 Iter 25600, Minibatch Loss= 0.297496, Training Accuracy= 0.91406 Iter 26880, Minibatch Loss= 0.274186, Training Accuracy= 0.89844 Iter 28160, Minibatch Loss= 0.153380, Training Accuracy= 0.94531 Iter 29440, Minibatch Loss= 0.223070, Training Accuracy= 0.93750 Iter 30720, Minibatch Loss= 0.187295, Training Accuracy= 0.92188 Iter 32000, Minibatch Loss= 0.192284, Training Accuracy= 0.94531 Iter 33280, Minibatch Loss= 0.206862, Training Accuracy= 0.93750 Iter 34560, Minibatch Loss= 0.157154, Training Accuracy= 0.94531 Iter 35840, Minibatch Loss= 0.221760, Training Accuracy= 0.92188 Iter 37120, Minibatch Loss= 0.204846, Training Accuracy= 0.94531 Iter 38400, Minibatch Loss= 0.143488, Training Accuracy= 0.96094 Iter 39680, Minibatch Loss= 0.147712, Training Accuracy= 0.95312 Iter 40960, Minibatch Loss= 0.260166, Training Accuracy= 0.92188 Iter 42240, Minibatch Loss= 0.236482, Training Accuracy= 0.92188 Iter 43520, Minibatch Loss= 0.222228, Training Accuracy= 0.93750 Iter 44800, Minibatch Loss= 0.135309, Training Accuracy= 0.96094 Iter 46080, Minibatch Loss= 0.148444, Training Accuracy= 0.96094 Iter 47360, Minibatch Loss= 0.284828, Training Accuracy= 0.92188 Iter 48640, Minibatch Loss= 0.276086, Training Accuracy= 0.92188 Iter 49920, Minibatch Loss= 0.111534, Training Accuracy= 0.95312 Iter 51200, Minibatch Loss= 0.111628, Training Accuracy= 0.96094 Iter 52480, Minibatch Loss= 0.131188, Training Accuracy= 0.96875 Iter 53760, Minibatch Loss= 0.080626, Training Accuracy= 0.98438 Iter 55040, Minibatch Loss= 0.118804, Training Accuracy= 0.96094 Iter 56320, Minibatch Loss= 0.211226, Training Accuracy= 0.92969 Iter 57600, Minibatch Loss= 0.165107, Training Accuracy= 0.95312 Iter 58880, Minibatch Loss= 0.134429, Training Accuracy= 0.96094 Iter 60160, Minibatch Loss= 0.168475, Training Accuracy= 0.95312 Iter 61440, Minibatch Loss= 0.128811, Training Accuracy= 0.96094 Iter 62720, Minibatch Loss= 0.077244, Training Accuracy= 0.96094 Iter 64000, Minibatch Loss= 0.090807, Training Accuracy= 0.96875 Iter 65280, Minibatch Loss= 0.192585, Training Accuracy= 0.92188 Iter 66560, Minibatch Loss= 0.197945, Training Accuracy= 0.95312 Iter 67840, Minibatch Loss= 0.146314, Training Accuracy= 0.96875 Iter 69120, Minibatch Loss= 0.117953, Training Accuracy= 0.96875 Iter 70400, Minibatch Loss= 0.174820, Training Accuracy= 0.96875 Iter 71680, Minibatch Loss= 0.078789, Training Accuracy= 0.98438 Iter 72960, Minibatch Loss= 0.180256, Training Accuracy= 0.92188 Iter 74240, Minibatch Loss= 0.142464, Training Accuracy= 0.94531 Iter 75520, Minibatch Loss= 0.112260, Training Accuracy= 0.96875 Iter 76800, Minibatch Loss= 0.133794, Training Accuracy= 0.95312 Iter 78080, Minibatch Loss= 0.067401, Training Accuracy= 0.99219 Iter 79360, Minibatch Loss= 0.154360, Training Accuracy= 0.96094 Iter 80640, Minibatch Loss= 0.067086, Training Accuracy= 0.97656 Iter 81920, Minibatch Loss= 0.110198, Training Accuracy= 0.97656 Iter 83200, Minibatch Loss= 0.149606, Training Accuracy= 0.95312 Iter 84480, Minibatch Loss= 0.119215, Training Accuracy= 0.96875 Iter 85760, Minibatch Loss= 0.051235, Training Accuracy= 0.98438 Iter 87040, Minibatch Loss= 0.094870, Training Accuracy= 0.96875 Iter 88320, Minibatch Loss= 0.073898, Training Accuracy= 0.98438 Iter 89600, Minibatch Loss= 0.087323, Training Accuracy= 0.96094 Iter 90880, Minibatch Loss= 0.088403, Training Accuracy= 0.96094 Iter 92160, Minibatch Loss= 0.120743, Training Accuracy= 0.97656 Iter 93440, Minibatch Loss= 0.061005, Training Accuracy= 0.98438 Iter 94720, Minibatch Loss= 0.095413, Training Accuracy= 0.97656 Iter 96000, Minibatch Loss= 0.078382, Training Accuracy= 0.97656 Iter 97280, Minibatch Loss= 0.100441, Training Accuracy= 0.97656 Iter 98560, Minibatch Loss= 0.022794, Training Accuracy= 1.00000 Iter 99840, Minibatch Loss= 0.085918, Training Accuracy= 0.97656 Finished! Testing Accuracy: 0.984375
实例60:静态多层LSTM对MNIST数据集分类
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
stacked_rnn = []
for i in range(3):
stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.892138, Training Accuracy= 0.27344 Iter 2560, Minibatch Loss= 1.520981, Training Accuracy= 0.42969 Iter 3840, Minibatch Loss= 1.271327, Training Accuracy= 0.53906 Iter 5120, Minibatch Loss= 0.980068, Training Accuracy= 0.65625 Iter 6400, Minibatch Loss= 0.812825, Training Accuracy= 0.74219 Iter 7680, Minibatch Loss= 0.807655, Training Accuracy= 0.74219 Iter 8960, Minibatch Loss= 0.761637, Training Accuracy= 0.77344 Iter 10240, Minibatch Loss= 0.580690, Training Accuracy= 0.82812 Iter 11520, Minibatch Loss= 0.802299, Training Accuracy= 0.72656 Iter 12800, Minibatch Loss= 0.557632, Training Accuracy= 0.82031 Iter 14080, Minibatch Loss= 0.580068, Training Accuracy= 0.85938 Iter 15360, Minibatch Loss= 0.365136, Training Accuracy= 0.88281 Iter 16640, Minibatch Loss= 0.554880, Training Accuracy= 0.83594 Iter 17920, Minibatch Loss= 0.392231, Training Accuracy= 0.88281 Iter 19200, Minibatch Loss= 0.239315, Training Accuracy= 0.92969 Iter 20480, Minibatch Loss= 0.266329, Training Accuracy= 0.92969 Iter 21760, Minibatch Loss= 0.239885, Training Accuracy= 0.93750 Iter 23040, Minibatch Loss= 0.308746, Training Accuracy= 0.89062 Iter 24320, Minibatch Loss= 0.405082, Training Accuracy= 0.86719 Iter 25600, Minibatch Loss= 0.306493, Training Accuracy= 0.92969 Iter 26880, Minibatch Loss= 0.297999, Training Accuracy= 0.92188 Iter 28160, Minibatch Loss= 0.205317, Training Accuracy= 0.94531 Iter 29440, Minibatch Loss= 0.209697, Training Accuracy= 0.92188 Iter 30720, Minibatch Loss= 0.224113, Training Accuracy= 0.92188 Iter 32000, Minibatch Loss= 0.202094, Training Accuracy= 0.93750 Iter 33280, Minibatch Loss= 0.209670, Training Accuracy= 0.92188 Iter 34560, Minibatch Loss= 0.275021, Training Accuracy= 0.92188 Iter 35840, Minibatch Loss= 0.141351, Training Accuracy= 0.95312 Iter 37120, Minibatch Loss= 0.172745, Training Accuracy= 0.92188 Iter 38400, Minibatch Loss= 0.228708, Training Accuracy= 0.92969 Iter 39680, Minibatch Loss= 0.245980, Training Accuracy= 0.92188 Iter 40960, Minibatch Loss= 0.216683, Training Accuracy= 0.92188 Iter 42240, Minibatch Loss= 0.183437, Training Accuracy= 0.96094 Iter 43520, Minibatch Loss= 0.249385, Training Accuracy= 0.92969 Iter 44800, Minibatch Loss= 0.275260, Training Accuracy= 0.91406 Iter 46080, Minibatch Loss= 0.174156, Training Accuracy= 0.92969 Iter 47360, Minibatch Loss= 0.261219, Training Accuracy= 0.90625 Iter 48640, Minibatch Loss= 0.187510, Training Accuracy= 0.93750 Iter 49920, Minibatch Loss= 0.090633, Training Accuracy= 0.98438 Iter 51200, Minibatch Loss= 0.061939, Training Accuracy= 0.97656 Iter 52480, Minibatch Loss= 0.075695, Training Accuracy= 0.97656 Iter 53760, Minibatch Loss= 0.322783, Training Accuracy= 0.92188 Iter 55040, Minibatch Loss= 0.126282, Training Accuracy= 0.96875 Iter 56320, Minibatch Loss= 0.119054, Training Accuracy= 0.96875 Iter 57600, Minibatch Loss= 0.213464, Training Accuracy= 0.94531 Iter 58880, Minibatch Loss= 0.083522, Training Accuracy= 0.96094 Iter 60160, Minibatch Loss= 0.094831, Training Accuracy= 0.98438 Iter 61440, Minibatch Loss= 0.096521, Training Accuracy= 0.96875 Iter 62720, Minibatch Loss= 0.173445, Training Accuracy= 0.93750 Iter 64000, Minibatch Loss= 0.119984, Training Accuracy= 0.96875 Iter 65280, Minibatch Loss= 0.180508, Training Accuracy= 0.95312 Iter 66560, Minibatch Loss= 0.068011, Training Accuracy= 0.98438 Iter 67840, Minibatch Loss= 0.210539, Training Accuracy= 0.95312 Iter 69120, Minibatch Loss= 0.097277, Training Accuracy= 0.96875 Iter 70400, Minibatch Loss= 0.196134, Training Accuracy= 0.95312 Iter 71680, Minibatch Loss= 0.091252, Training Accuracy= 0.96094 Iter 72960, Minibatch Loss= 0.144938, Training Accuracy= 0.95312 Iter 74240, Minibatch Loss= 0.101681, Training Accuracy= 0.96875 Iter 75520, Minibatch Loss= 0.057837, Training Accuracy= 0.97656 Iter 76800, Minibatch Loss= 0.095161, Training Accuracy= 0.97656 Iter 78080, Minibatch Loss= 0.091807, Training Accuracy= 0.98438 Iter 79360, Minibatch Loss= 0.063520, Training Accuracy= 0.99219 Iter 80640, Minibatch Loss= 0.087506, Training Accuracy= 0.96875 Iter 81920, Minibatch Loss= 0.264015, Training Accuracy= 0.93750 Iter 83200, Minibatch Loss= 0.163464, Training Accuracy= 0.95312 Iter 84480, Minibatch Loss= 0.074320, Training Accuracy= 0.98438 Iter 85760, Minibatch Loss= 0.087142, Training Accuracy= 0.96875 Iter 87040, Minibatch Loss= 0.061768, Training Accuracy= 0.97656 Iter 88320, Minibatch Loss= 0.129232, Training Accuracy= 0.97656 Iter 89600, Minibatch Loss= 0.054522, Training Accuracy= 0.98438 Iter 90880, Minibatch Loss= 0.060756, Training Accuracy= 0.97656 Iter 92160, Minibatch Loss= 0.034073, Training Accuracy= 1.00000 Iter 93440, Minibatch Loss= 0.112970, Training Accuracy= 0.97656 Iter 94720, Minibatch Loss= 0.088832, Training Accuracy= 0.96875 Iter 96000, Minibatch Loss= 0.093209, Training Accuracy= 0.96875 Iter 97280, Minibatch Loss= 0.098466, Training Accuracy= 0.97656 Iter 98560, Minibatch Loss= 0.085677, Training Accuracy= 0.97656 Iter 99840, Minibatch Loss= 0.102284, Training Accuracy= 0.96875 Finished! Testing Accuracy: 1.0
实例61:静态多层RNN-LSTM连接GRU对MNIST数据集分类
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
#
#stacked_rnn = []
#for i in range(3):
# stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
gru = tf.contrib.rnn.GRUCell(n_hidden*2)
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden)
mcell = tf.contrib.rnn.MultiRNNCell([lstm_cell,gru])
x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.808671, Training Accuracy= 0.42188 Iter 2560, Minibatch Loss= 1.288418, Training Accuracy= 0.59375 Iter 3840, Minibatch Loss= 0.954410, Training Accuracy= 0.75781 Iter 5120, Minibatch Loss= 0.873409, Training Accuracy= 0.70312 Iter 6400, Minibatch Loss= 0.872713, Training Accuracy= 0.66406 Iter 7680, Minibatch Loss= 0.618704, Training Accuracy= 0.83594 Iter 8960, Minibatch Loss= 0.426509, Training Accuracy= 0.88281 Iter 10240, Minibatch Loss= 0.480236, Training Accuracy= 0.86719 Iter 11520, Minibatch Loss= 0.429108, Training Accuracy= 0.84375 Iter 12800, Minibatch Loss= 0.552932, Training Accuracy= 0.82812 Iter 14080, Minibatch Loss= 0.362248, Training Accuracy= 0.87500 Iter 15360, Minibatch Loss= 0.265852, Training Accuracy= 0.92969 Iter 16640, Minibatch Loss= 0.315052, Training Accuracy= 0.87500 Iter 17920, Minibatch Loss= 0.259191, Training Accuracy= 0.93750 Iter 19200, Minibatch Loss= 0.360970, Training Accuracy= 0.89844 Iter 20480, Minibatch Loss= 0.292491, Training Accuracy= 0.94531 Iter 21760, Minibatch Loss= 0.259818, Training Accuracy= 0.90625 Iter 23040, Minibatch Loss= 0.293232, Training Accuracy= 0.93750 Iter 24320, Minibatch Loss= 0.244462, Training Accuracy= 0.94531 Iter 25600, Minibatch Loss= 0.227099, Training Accuracy= 0.93750 Iter 26880, Minibatch Loss= 0.231890, Training Accuracy= 0.92969 Iter 28160, Minibatch Loss= 0.196979, Training Accuracy= 0.94531 Iter 29440, Minibatch Loss= 0.165492, Training Accuracy= 0.95312 Iter 30720, Minibatch Loss= 0.252359, Training Accuracy= 0.95312 Iter 32000, Minibatch Loss= 0.237196, Training Accuracy= 0.94531 Iter 33280, Minibatch Loss= 0.197849, Training Accuracy= 0.95312 Iter 34560, Minibatch Loss= 0.126519, Training Accuracy= 0.96094 Iter 35840, Minibatch Loss= 0.173198, Training Accuracy= 0.95312 Iter 37120, Minibatch Loss= 0.176549, Training Accuracy= 0.95312 Iter 38400, Minibatch Loss= 0.206727, Training Accuracy= 0.95312 Iter 39680, Minibatch Loss= 0.170020, Training Accuracy= 0.94531 Iter 40960, Minibatch Loss= 0.211021, Training Accuracy= 0.91406 Iter 42240, Minibatch Loss= 0.174285, Training Accuracy= 0.93750 Iter 43520, Minibatch Loss= 0.074847, Training Accuracy= 0.99219 Iter 44800, Minibatch Loss= 0.188610, Training Accuracy= 0.96875 Iter 46080, Minibatch Loss= 0.098489, Training Accuracy= 0.96875 Iter 47360, Minibatch Loss= 0.164228, Training Accuracy= 0.96094 Iter 48640, Minibatch Loss= 0.099976, Training Accuracy= 0.97656 Iter 49920, Minibatch Loss= 0.185240, Training Accuracy= 0.93750 Iter 51200, Minibatch Loss= 0.146018, Training Accuracy= 0.95312 Iter 52480, Minibatch Loss= 0.085702, Training Accuracy= 0.97656 Iter 53760, Minibatch Loss= 0.079165, Training Accuracy= 0.98438 Iter 55040, Minibatch Loss= 0.230488, Training Accuracy= 0.94531 Iter 56320, Minibatch Loss= 0.026857, Training Accuracy= 1.00000 Iter 57600, Minibatch Loss= 0.189286, Training Accuracy= 0.93750 Iter 58880, Minibatch Loss= 0.085770, Training Accuracy= 0.97656 Iter 60160, Minibatch Loss= 0.102213, Training Accuracy= 0.96094 Iter 61440, Minibatch Loss= 0.087951, Training Accuracy= 0.97656 Iter 62720, Minibatch Loss= 0.105146, Training Accuracy= 0.96094 Iter 64000, Minibatch Loss= 0.150521, Training Accuracy= 0.96094 Iter 65280, Minibatch Loss= 0.051553, Training Accuracy= 0.99219 Iter 66560, Minibatch Loss= 0.044942, Training Accuracy= 0.98438 Iter 67840, Minibatch Loss= 0.093745, Training Accuracy= 0.95312 Iter 69120, Minibatch Loss= 0.091557, Training Accuracy= 0.96094 Iter 70400, Minibatch Loss= 0.079020, Training Accuracy= 0.97656 Iter 71680, Minibatch Loss= 0.107035, Training Accuracy= 0.96875 Iter 72960, Minibatch Loss= 0.103098, Training Accuracy= 0.96094 Iter 74240, Minibatch Loss= 0.137766, Training Accuracy= 0.96875 Iter 75520, Minibatch Loss= 0.087706, Training Accuracy= 0.96875 Iter 76800, Minibatch Loss= 0.056394, Training Accuracy= 0.99219 Iter 78080, Minibatch Loss= 0.097080, Training Accuracy= 0.97656 Iter 79360, Minibatch Loss= 0.064108, Training Accuracy= 0.97656 Iter 80640, Minibatch Loss= 0.081608, Training Accuracy= 0.97656 Iter 81920, Minibatch Loss= 0.074628, Training Accuracy= 0.98438 Iter 83200, Minibatch Loss= 0.173194, Training Accuracy= 0.96094 Iter 84480, Minibatch Loss= 0.166146, Training Accuracy= 0.94531 Iter 85760, Minibatch Loss= 0.132719, Training Accuracy= 0.97656 Iter 87040, Minibatch Loss= 0.119572, Training Accuracy= 0.96875 Iter 88320, Minibatch Loss= 0.030019, Training Accuracy= 1.00000 Iter 89600, Minibatch Loss= 0.176066, Training Accuracy= 0.96875 Iter 90880, Minibatch Loss= 0.069413, Training Accuracy= 0.96875 Iter 92160, Minibatch Loss= 0.112738, Training Accuracy= 0.97656 Iter 93440, Minibatch Loss= 0.069958, Training Accuracy= 0.98438 Iter 94720, Minibatch Loss= 0.037432, Training Accuracy= 0.99219 Iter 96000, Minibatch Loss= 0.082820, Training Accuracy= 0.98438 Iter 97280, Minibatch Loss= 0.121944, Training Accuracy= 0.96875 Iter 98560, Minibatch Loss= 0.058363, Training Accuracy= 0.99219 Iter 99840, Minibatch Loss= 0.139464, Training Accuracy= 0.96875 Finished! Testing Accuracy: 0.97
实例62:动态多层RNN对MNIST分类
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
#
#stacked_rnn = []
#for i in range(3):
# stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
gru = tf.contrib.rnn.GRUCell(n_hidden*2)
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden)
mcell = tf.contrib.rnn.MultiRNNCell([lstm_cell,gru])
#x1 = tf.unstack(x, n_steps, 1)
#outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
#pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
outputs,states = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序,取最后一个时序outputs[-1]=(?,256)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.786891, Training Accuracy= 0.36719 Iter 2560, Minibatch Loss= 1.360941, Training Accuracy= 0.55469 Iter 3840, Minibatch Loss= 1.208105, Training Accuracy= 0.60156 Iter 5120, Minibatch Loss= 1.075533, Training Accuracy= 0.62500 Iter 6400, Minibatch Loss= 0.780158, Training Accuracy= 0.72656 Iter 7680, Minibatch Loss= 0.785984, Training Accuracy= 0.71875 Iter 8960, Minibatch Loss= 0.459019, Training Accuracy= 0.87500 Iter 10240, Minibatch Loss= 0.699168, Training Accuracy= 0.78125 Iter 11520, Minibatch Loss= 0.549069, Training Accuracy= 0.82031 Iter 12800, Minibatch Loss= 0.412937, Training Accuracy= 0.83594 Iter 14080, Minibatch Loss= 0.359638, Training Accuracy= 0.88281 Iter 15360, Minibatch Loss= 0.358867, Training Accuracy= 0.89844 Iter 16640, Minibatch Loss= 0.384718, Training Accuracy= 0.85938 Iter 17920, Minibatch Loss= 0.344854, Training Accuracy= 0.90625 Iter 19200, Minibatch Loss= 0.320385, Training Accuracy= 0.89062 Iter 20480, Minibatch Loss= 0.424948, Training Accuracy= 0.89062 Iter 21760, Minibatch Loss= 0.237800, Training Accuracy= 0.92969 Iter 23040, Minibatch Loss= 0.153665, Training Accuracy= 0.96094 Iter 24320, Minibatch Loss= 0.269827, Training Accuracy= 0.92188 Iter 25600, Minibatch Loss= 0.209402, Training Accuracy= 0.94531 Iter 26880, Minibatch Loss= 0.191818, Training Accuracy= 0.96094 Iter 28160, Minibatch Loss= 0.222970, Training Accuracy= 0.91406 Iter 29440, Minibatch Loss= 0.114014, Training Accuracy= 0.98438 Iter 30720, Minibatch Loss= 0.237388, Training Accuracy= 0.92969 Iter 32000, Minibatch Loss= 0.107080, Training Accuracy= 0.97656 Iter 33280, Minibatch Loss= 0.160952, Training Accuracy= 0.94531 Iter 34560, Minibatch Loss= 0.153807, Training Accuracy= 0.95312 Iter 35840, Minibatch Loss= 0.162314, Training Accuracy= 0.93750 Iter 37120, Minibatch Loss= 0.158119, Training Accuracy= 0.94531 Iter 38400, Minibatch Loss= 0.075459, Training Accuracy= 0.97656 Iter 39680, Minibatch Loss= 0.145804, Training Accuracy= 0.96875 Iter 40960, Minibatch Loss= 0.099279, Training Accuracy= 0.99219 Iter 42240, Minibatch Loss= 0.171049, Training Accuracy= 0.94531 Iter 43520, Minibatch Loss= 0.231703, Training Accuracy= 0.93750 Iter 44800, Minibatch Loss= 0.102913, Training Accuracy= 0.97656 Iter 46080, Minibatch Loss= 0.091480, Training Accuracy= 0.98438 Iter 47360, Minibatch Loss= 0.101064, Training Accuracy= 0.96875 Iter 48640, Minibatch Loss= 0.187488, Training Accuracy= 0.94531 Iter 49920, Minibatch Loss= 0.115087, Training Accuracy= 0.96094 Iter 51200, Minibatch Loss= 0.059695, Training Accuracy= 0.99219 Iter 52480, Minibatch Loss= 0.112815, Training Accuracy= 0.95312 Iter 53760, Minibatch Loss= 0.177814, Training Accuracy= 0.94531 Iter 55040, Minibatch Loss= 0.082340, Training Accuracy= 0.98438 Iter 56320, Minibatch Loss= 0.110558, Training Accuracy= 0.96875 Iter 57600, Minibatch Loss= 0.103817, Training Accuracy= 0.97656 Iter 58880, Minibatch Loss= 0.084542, Training Accuracy= 0.97656 Iter 60160, Minibatch Loss= 0.173816, Training Accuracy= 0.95312 Iter 61440, Minibatch Loss= 0.084009, Training Accuracy= 0.96875 Iter 62720, Minibatch Loss= 0.103503, Training Accuracy= 0.96875 Iter 64000, Minibatch Loss= 0.054017, Training Accuracy= 0.97656 Iter 65280, Minibatch Loss= 0.040551, Training Accuracy= 0.99219 Iter 66560, Minibatch Loss= 0.156163, Training Accuracy= 0.95312 Iter 67840, Minibatch Loss= 0.082662, Training Accuracy= 0.96875 Iter 69120, Minibatch Loss= 0.038192, Training Accuracy= 1.00000 Iter 70400, Minibatch Loss= 0.167307, Training Accuracy= 0.96094 Iter 71680, Minibatch Loss= 0.112745, Training Accuracy= 0.97656 Iter 72960, Minibatch Loss= 0.053178, Training Accuracy= 0.98438 Iter 74240, Minibatch Loss= 0.150620, Training Accuracy= 0.95312 Iter 75520, Minibatch Loss= 0.086217, Training Accuracy= 0.98438 Iter 76800, Minibatch Loss= 0.108635, Training Accuracy= 0.98438 Iter 78080, Minibatch Loss= 0.078446, Training Accuracy= 0.98438 Iter 79360, Minibatch Loss= 0.117279, Training Accuracy= 0.96875 Iter 80640, Minibatch Loss= 0.046549, Training Accuracy= 0.97656 Iter 81920, Minibatch Loss= 0.082791, Training Accuracy= 0.97656 Iter 83200, Minibatch Loss= 0.047810, Training Accuracy= 0.98438 Iter 84480, Minibatch Loss= 0.048099, Training Accuracy= 0.99219 Iter 85760, Minibatch Loss= 0.109521, Training Accuracy= 0.96094 Iter 87040, Minibatch Loss= 0.054557, Training Accuracy= 0.98438 Iter 88320, Minibatch Loss= 0.035168, Training Accuracy= 0.99219 Iter 89600, Minibatch Loss= 0.077837, Training Accuracy= 0.98438 Iter 90880, Minibatch Loss= 0.162258, Training Accuracy= 0.94531 Iter 92160, Minibatch Loss= 0.121832, Training Accuracy= 0.97656 Iter 93440, Minibatch Loss= 0.040085, Training Accuracy= 0.98438 Iter 94720, Minibatch Loss= 0.047824, Training Accuracy= 0.98438 Iter 96000, Minibatch Loss= 0.112367, Training Accuracy= 0.96094 Iter 97280, Minibatch Loss= 0.077551, Training Accuracy= 0.99219 Iter 98560, Minibatch Loss= 0.093468, Training Accuracy= 0.95312 Iter 99840, Minibatch Loss= 0.088767, Training Accuracy= 0.98438 Finished! Testing Accuracy: 0.99
实例63:构建单层动态BiRNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# 反向cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,lstm_bw_cell,x,
dtype=tf.float32)
print(len(outputs),outputs[0].shape,outputs[1].shape)
outputs = tf.concat(outputs, 2)
outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.160943, Training Accuracy= 0.27344 Iter 2560, Minibatch Loss= 1.997462, Training Accuracy= 0.33594 Iter 3840, Minibatch Loss= 1.570268, Training Accuracy= 0.42188 Iter 5120, Minibatch Loss= 1.373489, Training Accuracy= 0.48438 Iter 6400, Minibatch Loss= 1.088359, Training Accuracy= 0.65625 Iter 7680, Minibatch Loss= 0.993419, Training Accuracy= 0.70312 Iter 8960, Minibatch Loss= 0.906906, Training Accuracy= 0.72656 Iter 10240, Minibatch Loss= 0.781511, Training Accuracy= 0.73438 Iter 11520, Minibatch Loss= 0.745537, Training Accuracy= 0.74219 Iter 12800, Minibatch Loss= 0.687559, Training Accuracy= 0.79688 Iter 14080, Minibatch Loss= 0.536104, Training Accuracy= 0.83594 Iter 15360, Minibatch Loss= 0.499159, Training Accuracy= 0.85938 Iter 16640, Minibatch Loss= 0.525461, Training Accuracy= 0.84375 Iter 17920, Minibatch Loss= 0.458517, Training Accuracy= 0.87500 Iter 19200, Minibatch Loss= 0.525781, Training Accuracy= 0.82031 Iter 20480, Minibatch Loss= 0.398223, Training Accuracy= 0.89062 Iter 21760, Minibatch Loss= 0.343273, Training Accuracy= 0.89062 Iter 23040, Minibatch Loss= 0.294032, Training Accuracy= 0.90625 Iter 24320, Minibatch Loss= 0.333711, Training Accuracy= 0.91406 Iter 25600, Minibatch Loss= 0.303608, Training Accuracy= 0.89062 Iter 26880, Minibatch Loss= 0.356443, Training Accuracy= 0.89844 Iter 28160, Minibatch Loss= 0.318978, Training Accuracy= 0.90625 Iter 29440, Minibatch Loss= 0.346513, Training Accuracy= 0.91406 Iter 30720, Minibatch Loss= 0.351226, Training Accuracy= 0.89062 Iter 32000, Minibatch Loss= 0.262708, Training Accuracy= 0.90625 Iter 33280, Minibatch Loss= 0.250729, Training Accuracy= 0.91406 Iter 34560, Minibatch Loss= 0.429823, Training Accuracy= 0.85156 Iter 35840, Minibatch Loss= 0.348004, Training Accuracy= 0.91406 Iter 37120, Minibatch Loss= 0.325374, Training Accuracy= 0.91406 Iter 38400, Minibatch Loss= 0.247374, Training Accuracy= 0.89062 Iter 39680, Minibatch Loss= 0.402448, Training Accuracy= 0.87500 Iter 40960, Minibatch Loss= 0.314637, Training Accuracy= 0.92188 Iter 42240, Minibatch Loss= 0.252825, Training Accuracy= 0.92969 Iter 43520, Minibatch Loss= 0.280104, Training Accuracy= 0.91406 Iter 44800, Minibatch Loss= 0.251025, Training Accuracy= 0.92969 Iter 46080, Minibatch Loss= 0.336563, Training Accuracy= 0.89062 Iter 47360, Minibatch Loss= 0.195152, Training Accuracy= 0.94531 Iter 48640, Minibatch Loss= 0.202264, Training Accuracy= 0.94531 Iter 49920, Minibatch Loss= 0.203700, Training Accuracy= 0.92188 Iter 51200, Minibatch Loss= 0.207661, Training Accuracy= 0.96094 Iter 52480, Minibatch Loss= 0.264310, Training Accuracy= 0.92969 Iter 53760, Minibatch Loss= 0.190808, Training Accuracy= 0.94531 Iter 55040, Minibatch Loss= 0.083677, Training Accuracy= 0.97656 Iter 56320, Minibatch Loss= 0.236587, Training Accuracy= 0.92188 Iter 57600, Minibatch Loss= 0.242121, Training Accuracy= 0.92969 Iter 58880, Minibatch Loss= 0.182118, Training Accuracy= 0.96094 Iter 60160, Minibatch Loss= 0.174736, Training Accuracy= 0.94531 Iter 61440, Minibatch Loss= 0.178520, Training Accuracy= 0.96875 Iter 62720, Minibatch Loss= 0.163360, Training Accuracy= 0.93750 Iter 64000, Minibatch Loss= 0.219186, Training Accuracy= 0.92969 Iter 65280, Minibatch Loss= 0.082264, Training Accuracy= 0.98438 Iter 66560, Minibatch Loss= 0.171661, Training Accuracy= 0.95312 Iter 67840, Minibatch Loss= 0.210177, Training Accuracy= 0.96094 Iter 69120, Minibatch Loss= 0.301095, Training Accuracy= 0.92969 Iter 70400, Minibatch Loss= 0.082820, Training Accuracy= 0.97656 Iter 71680, Minibatch Loss= 0.222693, Training Accuracy= 0.93750 Iter 72960, Minibatch Loss= 0.164202, Training Accuracy= 0.92969 Iter 74240, Minibatch Loss= 0.158589, Training Accuracy= 0.95312 Iter 75520, Minibatch Loss= 0.155147, Training Accuracy= 0.96094 Iter 76800, Minibatch Loss= 0.139993, Training Accuracy= 0.95312 Iter 78080, Minibatch Loss= 0.138517, Training Accuracy= 0.95312 Iter 79360, Minibatch Loss= 0.184054, Training Accuracy= 0.94531 Iter 80640, Minibatch Loss= 0.207642, Training Accuracy= 0.92188 Iter 81920, Minibatch Loss= 0.127576, Training Accuracy= 0.96094 Iter 83200, Minibatch Loss= 0.157001, Training Accuracy= 0.96094 Iter 84480, Minibatch Loss= 0.169914, Training Accuracy= 0.94531 Iter 85760, Minibatch Loss= 0.133477, Training Accuracy= 0.96094 Iter 87040, Minibatch Loss= 0.244336, Training Accuracy= 0.89844 Iter 88320, Minibatch Loss= 0.071679, Training Accuracy= 0.99219Iter 89600, Minibatch Loss= 0.123343, Training Accuracy= 0.96094 Iter 90880, Minibatch Loss= 0.217972, Training Accuracy= 0.92188 Iter 92160, Minibatch Loss= 0.082326, Training Accuracy= 0.99219 Iter 93440, Minibatch Loss= 0.206144, Training Accuracy= 0.94531 Iter 94720, Minibatch Loss= 0.119195, Training Accuracy= 0.97656 Iter 96000, Minibatch Loss= 0.076865, Training Accuracy= 0.97656 Iter 97280, Minibatch Loss= 0.208217, Training Accuracy= 0.92969 Iter 98560, Minibatch Loss= 0.143691, Training Accuracy= 0.94531 Iter 99840, Minibatch Loss= 0.119417, Training Accuracy= 0.97656 Finished! Testing Accuracy: 0.9765625
实例64:构建单层静态BiRNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# 反向cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
dtype=tf.float32)
print(outputs[0].shape,len(outputs))
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.198292, Training Accuracy= 0.34375 Iter 2560, Minibatch Loss= 1.922280, Training Accuracy= 0.33594 Iter 3840, Minibatch Loss= 1.619660, Training Accuracy= 0.48438 Iter 5120, Minibatch Loss= 1.287644, Training Accuracy= 0.57812 Iter 6400, Minibatch Loss= 1.054544, Training Accuracy= 0.63281 Iter 7680, Minibatch Loss= 0.970391, Training Accuracy= 0.64844 Iter 8960, Minibatch Loss= 0.785846, Training Accuracy= 0.75000 Iter 10240, Minibatch Loss= 0.735042, Training Accuracy= 0.77344 Iter 11520, Minibatch Loss= 0.709633, Training Accuracy= 0.78906 Iter 12800, Minibatch Loss= 0.494802, Training Accuracy= 0.85938 Iter 14080, Minibatch Loss= 0.587061, Training Accuracy= 0.82812 Iter 15360, Minibatch Loss= 0.571160, Training Accuracy= 0.80469 Iter 16640, Minibatch Loss= 0.576181, Training Accuracy= 0.80469 Iter 17920, Minibatch Loss= 0.425966, Training Accuracy= 0.89062 Iter 19200, Minibatch Loss= 0.334115, Training Accuracy= 0.91406 Iter 20480, Minibatch Loss= 0.367955, Training Accuracy= 0.89844 Iter 21760, Minibatch Loss= 0.314265, Training Accuracy= 0.90625 Iter 23040, Minibatch Loss= 0.325072, Training Accuracy= 0.89844 Iter 24320, Minibatch Loss= 0.427572, Training Accuracy= 0.86719 Iter 25600, Minibatch Loss= 0.407501, Training Accuracy= 0.86719 Iter 26880, Minibatch Loss= 0.279054, Training Accuracy= 0.92969 Iter 28160, Minibatch Loss= 0.250573, Training Accuracy= 0.91406 Iter 29440, Minibatch Loss= 0.243975, Training Accuracy= 0.92188 Iter 30720, Minibatch Loss= 0.238282, Training Accuracy= 0.93750 Iter 32000, Minibatch Loss= 0.205076, Training Accuracy= 0.92188 Iter 33280, Minibatch Loss= 0.346559, Training Accuracy= 0.89844 Iter 34560, Minibatch Loss= 0.247907, Training Accuracy= 0.92188 Iter 35840, Minibatch Loss= 0.277912, Training Accuracy= 0.90625 Iter 37120, Minibatch Loss= 0.290173, Training Accuracy= 0.92188 Iter 38400, Minibatch Loss= 0.301018, Training Accuracy= 0.89062 Iter 39680, Minibatch Loss= 0.269850, Training Accuracy= 0.90625 Iter 40960, Minibatch Loss= 0.276790, Training Accuracy= 0.91406 Iter 42240, Minibatch Loss= 0.234881, Training Accuracy= 0.95312 Iter 43520, Minibatch Loss= 0.276319, Training Accuracy= 0.91406 Iter 44800, Minibatch Loss= 0.232567, Training Accuracy= 0.92188 Iter 46080, Minibatch Loss= 0.319510, Training Accuracy= 0.89062 Iter 47360, Minibatch Loss= 0.148028, Training Accuracy= 0.95312 Iter 48640, Minibatch Loss= 0.176917, Training Accuracy= 0.95312 Iter 49920, Minibatch Loss= 0.236975, Training Accuracy= 0.92969 Iter 51200, Minibatch Loss= 0.129818, Training Accuracy= 0.96094 Iter 52480, Minibatch Loss= 0.238299, Training Accuracy= 0.94531 Iter 53760, Minibatch Loss= 0.156861, Training Accuracy= 0.95312 Iter 55040, Minibatch Loss= 0.216879, Training Accuracy= 0.92969 Iter 56320, Minibatch Loss= 0.302488, Training Accuracy= 0.89844 Iter 57600, Minibatch Loss= 0.131714, Training Accuracy= 0.96094 Iter 58880, Minibatch Loss= 0.111539, Training Accuracy= 0.98438 Iter 60160, Minibatch Loss= 0.153051, Training Accuracy= 0.94531 Iter 61440, Minibatch Loss= 0.203167, Training Accuracy= 0.93750 Iter 62720, Minibatch Loss= 0.135487, Training Accuracy= 0.96875 Iter 64000, Minibatch Loss= 0.193364, Training Accuracy= 0.93750 Iter 65280, Minibatch Loss= 0.253839, Training Accuracy= 0.92969 Iter 66560, Minibatch Loss= 0.167367, Training Accuracy= 0.92969 Iter 67840, Minibatch Loss= 0.292977, Training Accuracy= 0.92188 Iter 69120, Minibatch Loss= 0.167282, Training Accuracy= 0.95312 Iter 70400, Minibatch Loss= 0.320158, Training Accuracy= 0.90625 Iter 71680, Minibatch Loss= 0.104524, Training Accuracy= 0.96875 Iter 72960, Minibatch Loss= 0.150813, Training Accuracy= 0.95312 Iter 74240, Minibatch Loss= 0.221704, Training Accuracy= 0.94531 Iter 75520, Minibatch Loss= 0.163385, Training Accuracy= 0.96094 Iter 76800, Minibatch Loss= 0.116737, Training Accuracy= 0.96875 Iter 78080, Minibatch Loss= 0.125065, Training Accuracy= 0.96094 Iter 79360, Minibatch Loss= 0.297941, Training Accuracy= 0.92188 Iter 80640, Minibatch Loss= 0.195890, Training Accuracy= 0.93750 Iter 81920, Minibatch Loss= 0.071490, Training Accuracy= 0.99219 Iter 83200, Minibatch Loss= 0.115205, Training Accuracy= 0.97656 Iter 84480, Minibatch Loss= 0.046660, Training Accuracy= 0.99219 Iter 85760, Minibatch Loss= 0.104727, Training Accuracy= 0.96875 Iter 87040, Minibatch Loss= 0.126642, Training Accuracy= 0.94531 Iter 88320, Minibatch Loss= 0.112083, Training Accuracy= 0.97656 Iter 89600, Minibatch Loss= 0.084982, Training Accuracy= 0.98438 Iter 90880, Minibatch Loss= 0.184768, Training Accuracy= 0.94531 Iter 92160, Minibatch Loss= 0.192093, Training Accuracy= 0.94531 Iter 93440, Minibatch Loss= 0.108329, Training Accuracy= 0.98438 Iter 94720, Minibatch Loss= 0.102838, Training Accuracy= 0.97656 Iter 96000, Minibatch Loss= 0.065941, Training Accuracy= 0.98438 Iter 97280, Minibatch Loss= 0.138307, Training Accuracy= 0.96094 Iter 98560, Minibatch Loss= 0.106404, Training Accuracy= 0.95312 Iter 99840, Minibatch Loss= 0.184734, Training Accuracy= 0.93750 Finished! Testing Accuracy: 0.9765625
实例65:构建多层BiRNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
# dtype=tf.float32)
outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
dtype=tf.float32)
print(outputs[0].shape,len(outputs))
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 2.172426, Training Accuracy= 0.36719 Iter 2560, Minibatch Loss= 1.861094, Training Accuracy= 0.30469 Iter 3840, Minibatch Loss= 1.509905, Training Accuracy= 0.53125 Iter 5120, Minibatch Loss= 1.420893, Training Accuracy= 0.52344 Iter 6400, Minibatch Loss= 1.035572, Training Accuracy= 0.65625 Iter 7680, Minibatch Loss= 0.857329, Training Accuracy= 0.69531 Iter 8960, Minibatch Loss= 0.792173, Training Accuracy= 0.75000 Iter 10240, Minibatch Loss= 0.736671, Training Accuracy= 0.77344 Iter 11520, Minibatch Loss= 0.735905, Training Accuracy= 0.77344 Iter 12800, Minibatch Loss= 0.661815, Training Accuracy= 0.80469 Iter 14080, Minibatch Loss= 0.606804, Training Accuracy= 0.77344 Iter 15360, Minibatch Loss= 0.560650, Training Accuracy= 0.81250 Iter 16640, Minibatch Loss= 0.492422, Training Accuracy= 0.84375 Iter 17920, Minibatch Loss= 0.556491, Training Accuracy= 0.83594 Iter 19200, Minibatch Loss= 0.479821, Training Accuracy= 0.83594 Iter 20480, Minibatch Loss= 0.519516, Training Accuracy= 0.80469 Iter 21760, Minibatch Loss= 0.408189, Training Accuracy= 0.86719 Iter 23040, Minibatch Loss= 0.294251, Training Accuracy= 0.89844 Iter 24320, Minibatch Loss= 0.237520, Training Accuracy= 0.92188 Iter 25600, Minibatch Loss= 0.510871, Training Accuracy= 0.87500 Iter 26880, Minibatch Loss= 0.335866, Training Accuracy= 0.89844 Iter 28160, Minibatch Loss= 0.234492, Training Accuracy= 0.92188 Iter 29440, Minibatch Loss= 0.294053, Training Accuracy= 0.92188 Iter 30720, Minibatch Loss= 0.268390, Training Accuracy= 0.91406 Iter 32000, Minibatch Loss= 0.233152, Training Accuracy= 0.92188 Iter 33280, Minibatch Loss= 0.252517, Training Accuracy= 0.92969 Iter 34560, Minibatch Loss= 0.280433, Training Accuracy= 0.92188 Iter 35840, Minibatch Loss= 0.430729, Training Accuracy= 0.86719 Iter 37120, Minibatch Loss= 0.330445, Training Accuracy= 0.92969 Iter 38400, Minibatch Loss= 0.234638, Training Accuracy= 0.92969 Iter 39680, Minibatch Loss= 0.223448, Training Accuracy= 0.95312 Iter 40960, Minibatch Loss= 0.288984, Training Accuracy= 0.89844 Iter 42240, Minibatch Loss= 0.223206, Training Accuracy= 0.92969 Iter 43520, Minibatch Loss= 0.204564, Training Accuracy= 0.95312 Iter 44800, Minibatch Loss= 0.137583, Training Accuracy= 0.97656 Iter 46080, Minibatch Loss= 0.172719, Training Accuracy= 0.95312 Iter 47360, Minibatch Loss= 0.181917, Training Accuracy= 0.95312 Iter 48640, Minibatch Loss= 0.245667, Training Accuracy= 0.93750 Iter 49920, Minibatch Loss= 0.171087, Training Accuracy= 0.94531 Iter 51200, Minibatch Loss= 0.273686, Training Accuracy= 0.92969 Iter 52480, Minibatch Loss= 0.110942, Training Accuracy= 0.96094 Iter 53760, Minibatch Loss= 0.252516, Training Accuracy= 0.92969 Iter 55040, Minibatch Loss= 0.184445, Training Accuracy= 0.96094 Iter 56320, Minibatch Loss= 0.212692, Training Accuracy= 0.93750 Iter 57600, Minibatch Loss= 0.148188, Training Accuracy= 0.94531 Iter 58880, Minibatch Loss= 0.202650, Training Accuracy= 0.93750 Iter 60160, Minibatch Loss= 0.162564, Training Accuracy= 0.95312 Iter 61440, Minibatch Loss= 0.140252, Training Accuracy= 0.96875 Iter 62720, Minibatch Loss= 0.080859, Training Accuracy= 0.98438 Iter 64000, Minibatch Loss= 0.271262, Training Accuracy= 0.92188 Iter 65280, Minibatch Loss= 0.130727, Training Accuracy= 0.96875 Iter 66560, Minibatch Loss= 0.202950, Training Accuracy= 0.94531 Iter 67840, Minibatch Loss= 0.104672, Training Accuracy= 0.97656 Iter 69120, Minibatch Loss= 0.104267, Training Accuracy= 0.96875 Iter 70400, Minibatch Loss= 0.264764, Training Accuracy= 0.91406 Iter 71680, Minibatch Loss= 0.247204, Training Accuracy= 0.92969 Iter 72960, Minibatch Loss= 0.114761, Training Accuracy= 0.96875 Iter 74240, Minibatch Loss= 0.127167, Training Accuracy= 0.96875 Iter 75520, Minibatch Loss= 0.150572, Training Accuracy= 0.95312 Iter 76800, Minibatch Loss= 0.266346, Training Accuracy= 0.92969 Iter 78080, Minibatch Loss= 0.188211, Training Accuracy= 0.92188 Iter 79360, Minibatch Loss= 0.178170, Training Accuracy= 0.94531 Iter 80640, Minibatch Loss= 0.065349, Training Accuracy= 0.99219 Iter 81920, Minibatch Loss= 0.114197, Training Accuracy= 0.96094 Iter 83200, Minibatch Loss= 0.166545, Training Accuracy= 0.94531 Iter 84480, Minibatch Loss= 0.170978, Training Accuracy= 0.96094 Iter 85760, Minibatch Loss= 0.100027, Training Accuracy= 0.98438 Iter 87040, Minibatch Loss= 0.068403, Training Accuracy= 0.97656 Iter 88320, Minibatch Loss= 0.084172, Training Accuracy= 0.97656 Iter 89600, Minibatch Loss= 0.109292, Training Accuracy= 0.96875 Iter 90880, Minibatch Loss= 0.142447, Training Accuracy= 0.96094 Iter 92160, Minibatch Loss= 0.153235, Training Accuracy= 0.94531 Iter 93440, Minibatch Loss= 0.244075, Training Accuracy= 0.93750 Iter 94720, Minibatch Loss= 0.181213, Training Accuracy= 0.96094 Iter 96000, Minibatch Loss= 0.112537, Training Accuracy= 0.97656 Iter 97280, Minibatch Loss= 0.081839, Training Accuracy= 0.97656 Iter 98560, Minibatch Loss= 0.052867, Training Accuracy= 0.99219 Iter 99840, Minibatch Loss= 0.153948, Training Accuracy= 0.96875 Finished! Testing Accuracy: 0.984375
list多层BiRNN
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
# dtype=tf.float32)
#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
# dtype=tf.float32)
stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):
stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,
dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.821590, Training Accuracy= 0.35156 Iter 2560, Minibatch Loss= 1.534426, Training Accuracy= 0.45312 Iter 3840, Minibatch Loss= 1.064070, Training Accuracy= 0.64062 Iter 5120, Minibatch Loss= 0.864886, Training Accuracy= 0.69531 Iter 6400, Minibatch Loss= 0.761668, Training Accuracy= 0.71094 Iter 7680, Minibatch Loss= 0.740874, Training Accuracy= 0.77344 Iter 8960, Minibatch Loss= 0.588481, Training Accuracy= 0.81250 Iter 10240, Minibatch Loss= 0.462982, Training Accuracy= 0.82812 Iter 11520, Minibatch Loss= 0.370633, Training Accuracy= 0.88281 Iter 12800, Minibatch Loss= 0.338352, Training Accuracy= 0.89062 Iter 14080, Minibatch Loss= 0.331004, Training Accuracy= 0.92188 Iter 15360, Minibatch Loss= 0.311066, Training Accuracy= 0.86719 Iter 16640, Minibatch Loss= 0.343783, Training Accuracy= 0.87500 Iter 17920, Minibatch Loss= 0.372595, Training Accuracy= 0.88281 Iter 19200, Minibatch Loss= 0.325485, Training Accuracy= 0.92188 Iter 20480, Minibatch Loss= 0.264375, Training Accuracy= 0.89844 Iter 21760, Minibatch Loss= 0.139555, Training Accuracy= 0.98438 Iter 23040, Minibatch Loss= 0.148688, Training Accuracy= 0.96094 Iter 24320, Minibatch Loss= 0.258720, Training Accuracy= 0.93750 Iter 25600, Minibatch Loss= 0.226550, Training Accuracy= 0.92188 Iter 26880, Minibatch Loss= 0.138458, Training Accuracy= 0.96875 Iter 28160, Minibatch Loss= 0.213251, Training Accuracy= 0.92188 Iter 29440, Minibatch Loss= 0.254001, Training Accuracy= 0.92188 Iter 30720, Minibatch Loss= 0.167976, Training Accuracy= 0.96875 Iter 32000, Minibatch Loss= 0.135351, Training Accuracy= 0.95312 Iter 33280, Minibatch Loss= 0.160028, Training Accuracy= 0.94531 Iter 34560, Minibatch Loss= 0.077145, Training Accuracy= 0.98438 Iter 35840, Minibatch Loss= 0.191825, Training Accuracy= 0.94531 Iter 37120, Minibatch Loss= 0.165540, Training Accuracy= 0.95312 Iter 38400, Minibatch Loss= 0.196213, Training Accuracy= 0.94531 Iter 39680, Minibatch Loss= 0.287433, Training Accuracy= 0.93750 Iter 40960, Minibatch Loss= 0.191275, Training Accuracy= 0.95312 Iter 42240, Minibatch Loss= 0.151342, Training Accuracy= 0.96094 Iter 43520, Minibatch Loss= 0.125097, Training Accuracy= 0.96875 Iter 44800, Minibatch Loss= 0.102935, Training Accuracy= 0.98438 Iter 46080, Minibatch Loss= 0.266253, Training Accuracy= 0.92969 Iter 47360, Minibatch Loss= 0.142998, Training Accuracy= 0.95312 Iter 48640, Minibatch Loss= 0.184273, Training Accuracy= 0.93750 Iter 49920, Minibatch Loss= 0.119624, Training Accuracy= 0.95312 Iter 51200, Minibatch Loss= 0.142909, Training Accuracy= 0.96875 Iter 52480, Minibatch Loss= 0.116738, Training Accuracy= 0.96875 Iter 53760, Minibatch Loss= 0.060568, Training Accuracy= 0.97656 Iter 55040, Minibatch Loss= 0.058581, Training Accuracy= 0.98438 Iter 56320, Minibatch Loss= 0.124698, Training Accuracy= 0.96094 Iter 57600, Minibatch Loss= 0.100896, Training Accuracy= 0.98438 Iter 58880, Minibatch Loss= 0.110977, Training Accuracy= 0.95312 Iter 60160, Minibatch Loss= 0.106460, Training Accuracy= 0.97656 Iter 61440, Minibatch Loss= 0.110304, Training Accuracy= 0.96875 Iter 62720, Minibatch Loss= 0.232378, Training Accuracy= 0.94531 Iter 64000, Minibatch Loss= 0.203506, Training Accuracy= 0.93750 Iter 65280, Minibatch Loss= 0.056710, Training Accuracy= 0.97656 Iter 66560, Minibatch Loss= 0.072753, Training Accuracy= 0.97656 Iter 67840, Minibatch Loss= 0.079578, Training Accuracy= 0.97656 Iter 69120, Minibatch Loss= 0.107077, Training Accuracy= 0.96094 Iter 70400, Minibatch Loss= 0.096304, Training Accuracy= 0.97656 Iter 71680, Minibatch Loss= 0.144363, Training Accuracy= 0.96094 Iter 72960, Minibatch Loss= 0.091259, Training Accuracy= 0.96094 Iter 74240, Minibatch Loss= 0.048933, Training Accuracy= 0.97656 Iter 75520, Minibatch Loss= 0.160002, Training Accuracy= 0.92969 Iter 76800, Minibatch Loss= 0.058088, Training Accuracy= 0.99219 Iter 78080, Minibatch Loss= 0.127986, Training Accuracy= 0.96094 Iter 79360, Minibatch Loss= 0.038077, Training Accuracy= 0.99219 Iter 80640, Minibatch Loss= 0.111614, Training Accuracy= 0.96875 Iter 81920, Minibatch Loss= 0.042832, Training Accuracy= 0.99219 Iter 83200, Minibatch Loss= 0.099700, Training Accuracy= 0.95312 Iter 84480, Minibatch Loss= 0.070158, Training Accuracy= 0.97656 Iter 85760, Minibatch Loss= 0.044101, Training Accuracy= 0.98438 Iter 87040, Minibatch Loss= 0.148444, Training Accuracy= 0.94531 Iter 88320, Minibatch Loss= 0.053288, Training Accuracy= 0.99219 Iter 89600, Minibatch Loss= 0.086438, Training Accuracy= 0.96875 Iter 90880, Minibatch Loss= 0.084410, Training Accuracy= 0.97656 Iter 92160, Minibatch Loss= 0.070647, Training Accuracy= 0.96875 Iter 93440, Minibatch Loss= 0.157281, Training Accuracy= 0.94531 Iter 94720, Minibatch Loss= 0.154711, Training Accuracy= 0.96875 Iter 96000, Minibatch Loss= 0.067240, Training Accuracy= 0.96875 Iter 97280, Minibatch Loss= 0.067891, Training Accuracy= 0.99219 Iter 98560, Minibatch Loss= 0.040220, Training Accuracy= 0.99219 Iter 99840, Minibatch Loss= 0.113512, Training Accuracy= 0.96094 Finished! Testing Accuracy: 0.96875
MultiBiRNN
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
# dtype=tf.float32)
#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
# dtype=tf.float32)
stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):
stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,
# dtype=tf.float32)
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
mcell_bw = tf.contrib.rnn.MultiRNNCell(stacked_bw_rnn)
outputs, _, _ = rnn.stack_bidirectional_rnn([mcell],[mcell_bw], x1,
dtype=tf.float32)
#outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn([mcell],[mcell_bw], x,
# dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
#print(outputs[0].shape,outputs.shape)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.969089, Training Accuracy= 0.28906 Iter 2560, Minibatch Loss= 1.607973, Training Accuracy= 0.40625 Iter 3840, Minibatch Loss= 1.223222, Training Accuracy= 0.59375 Iter 5120, Minibatch Loss= 1.054859, Training Accuracy= 0.64844 Iter 6400, Minibatch Loss= 0.956632, Training Accuracy= 0.71875 Iter 7680, Minibatch Loss= 0.862249, Training Accuracy= 0.70312 Iter 8960, Minibatch Loss= 0.752958, Training Accuracy= 0.77344 Iter 10240, Minibatch Loss= 0.697205, Training Accuracy= 0.77344 Iter 11520, Minibatch Loss= 0.563148, Training Accuracy= 0.79688 Iter 12800, Minibatch Loss= 0.559524, Training Accuracy= 0.82031 Iter 14080, Minibatch Loss= 0.394891, Training Accuracy= 0.87500 Iter 15360, Minibatch Loss= 0.651662, Training Accuracy= 0.79688 Iter 16640, Minibatch Loss= 0.458168, Training Accuracy= 0.85938 Iter 17920, Minibatch Loss= 0.525461, Training Accuracy= 0.84375 Iter 19200, Minibatch Loss= 0.323415, Training Accuracy= 0.89844 Iter 20480, Minibatch Loss= 0.407945, Training Accuracy= 0.90625 Iter 21760, Minibatch Loss= 0.342875, Training Accuracy= 0.90625 Iter 23040, Minibatch Loss= 0.278589, Training Accuracy= 0.92188 Iter 24320, Minibatch Loss= 0.248584, Training Accuracy= 0.91406 Iter 25600, Minibatch Loss= 0.314651, Training Accuracy= 0.88281 Iter 26880, Minibatch Loss= 0.256853, Training Accuracy= 0.90625 Iter 28160, Minibatch Loss= 0.305443, Training Accuracy= 0.92969 Iter 29440, Minibatch Loss= 0.370154, Training Accuracy= 0.90625 Iter 30720, Minibatch Loss= 0.262370, Training Accuracy= 0.92188 Iter 32000, Minibatch Loss= 0.239071, Training Accuracy= 0.92969 Iter 33280, Minibatch Loss= 0.277529, Training Accuracy= 0.90625 Iter 34560, Minibatch Loss= 0.147705, Training Accuracy= 0.96094 Iter 35840, Minibatch Loss= 0.210542, Training Accuracy= 0.92969 Iter 37120, Minibatch Loss= 0.221123, Training Accuracy= 0.92188 Iter 38400, Minibatch Loss= 0.357465, Training Accuracy= 0.92188 Iter 39680, Minibatch Loss= 0.164977, Training Accuracy= 0.95312 Iter 40960, Minibatch Loss= 0.243880, Training Accuracy= 0.92969 Iter 42240, Minibatch Loss= 0.213714, Training Accuracy= 0.94531 Iter 43520, Minibatch Loss= 0.133602, Training Accuracy= 0.96875 Iter 44800, Minibatch Loss= 0.304978, Training Accuracy= 0.90625 Iter 46080, Minibatch Loss= 0.228339, Training Accuracy= 0.92188 Iter 47360, Minibatch Loss= 0.240235, Training Accuracy= 0.92188 Iter 48640, Minibatch Loss= 0.154399, Training Accuracy= 0.96875 Iter 49920, Minibatch Loss= 0.252016, Training Accuracy= 0.92188 Iter 51200, Minibatch Loss= 0.171817, Training Accuracy= 0.96875 Iter 52480, Minibatch Loss= 0.145279, Training Accuracy= 0.96875 Iter 53760, Minibatch Loss= 0.164326, Training Accuracy= 0.93750 Iter 55040, Minibatch Loss= 0.099755, Training Accuracy= 0.96094 Iter 56320, Minibatch Loss= 0.085352, Training Accuracy= 0.97656 Iter 57600, Minibatch Loss= 0.172950, Training Accuracy= 0.95312 Iter 58880, Minibatch Loss= 0.123920, Training Accuracy= 0.96875 Iter 60160, Minibatch Loss= 0.115939, Training Accuracy= 0.97656 Iter 61440, Minibatch Loss= 0.084130, Training Accuracy= 0.97656 Iter 62720, Minibatch Loss= 0.149050, Training Accuracy= 0.93750 Iter 64000, Minibatch Loss= 0.168845, Training Accuracy= 0.95312 Iter 65280, Minibatch Loss= 0.140895, Training Accuracy= 0.96094 Iter 66560, Minibatch Loss= 0.126092, Training Accuracy= 0.95312 Iter 67840, Minibatch Loss= 0.147673, Training Accuracy= 0.96094 Iter 69120, Minibatch Loss= 0.057092, Training Accuracy= 0.99219 Iter 70400, Minibatch Loss= 0.037331, Training Accuracy= 1.00000 Iter 71680, Minibatch Loss= 0.188352, Training Accuracy= 0.95312 Iter 72960, Minibatch Loss= 0.108095, Training Accuracy= 0.96875 Iter 74240, Minibatch Loss= 0.059791, Training Accuracy= 0.99219 Iter 75520, Minibatch Loss= 0.113426, Training Accuracy= 0.96875 Iter 76800, Minibatch Loss= 0.133168, Training Accuracy= 0.97656 Iter 78080, Minibatch Loss= 0.183791, Training Accuracy= 0.95312 Iter 79360, Minibatch Loss= 0.122682, Training Accuracy= 0.96875 Iter 80640, Minibatch Loss= 0.057580, Training Accuracy= 0.97656 Iter 81920, Minibatch Loss= 0.167665, Training Accuracy= 0.94531 Iter 83200, Minibatch Loss= 0.122606, Training Accuracy= 0.96094 Iter 84480, Minibatch Loss= 0.091936, Training Accuracy= 0.98438 Iter 85760, Minibatch Loss= 0.171889, Training Accuracy= 0.95312 Iter 87040, Minibatch Loss= 0.117054, Training Accuracy= 0.96875 Iter 88320, Minibatch Loss= 0.051410, Training Accuracy= 0.97656 Iter 89600, Minibatch Loss= 0.071451, Training Accuracy= 0.97656 Iter 90880, Minibatch Loss= 0.048790, Training Accuracy= 0.99219 Iter 92160, Minibatch Loss= 0.123331, Training Accuracy= 0.96094 Iter 93440, Minibatch Loss= 0.044237, Training Accuracy= 0.96875 Iter 94720, Minibatch Loss= 0.115570, Training Accuracy= 0.96875 Iter 96000, Minibatch Loss= 0.103702, Training Accuracy= 0.97656 Iter 97280, Minibatch Loss= 0.088024, Training Accuracy= 0.99219 Iter 98560, Minibatch Loss= 0.071973, Training Accuracy= 0.98438 Iter 99840, Minibatch Loss= 0.065105, Training Accuracy= 0.99219 Finished! Testing Accuracy: 0.9765625
实例66:构建多层动态BiRNN对MNIST数据集分类
# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
# dtype=tf.float32)
#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
# dtype=tf.float32)
stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):
stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,
# dtype=tf.float32)
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
mcell_bw = tf.contrib.rnn.MultiRNNCell(stacked_bw_rnn)
#outputs, _, _ = rnn.stack_bidirectional_rnn([mcell],[mcell_bw], x1,
# dtype=tf.float32)
outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn([mcell],[mcell_bw], x,
dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2]) #batch_size, max_time, layers_output]`
print(outputs[0].shape,outputs.shape)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 1.925515, Training Accuracy= 0.28906 Iter 2560, Minibatch Loss= 1.478539, Training Accuracy= 0.47656 Iter 3840, Minibatch Loss= 1.367934, Training Accuracy= 0.59375 Iter 5120, Minibatch Loss= 1.002120, Training Accuracy= 0.67188 Iter 6400, Minibatch Loss= 0.918952, Training Accuracy= 0.64062 Iter 7680, Minibatch Loss= 0.822888, Training Accuracy= 0.70312 Iter 8960, Minibatch Loss= 0.806084, Training Accuracy= 0.72656 Iter 10240, Minibatch Loss= 0.769249, Training Accuracy= 0.75781 Iter 11520, Minibatch Loss= 0.657592, Training Accuracy= 0.75781 Iter 12800, Minibatch Loss= 0.659502, Training Accuracy= 0.78125 Iter 14080, Minibatch Loss= 0.473548, Training Accuracy= 0.83594 Iter 15360, Minibatch Loss= 0.301996, Training Accuracy= 0.92188 Iter 16640, Minibatch Loss= 0.417549, Training Accuracy= 0.85938 Iter 17920, Minibatch Loss= 0.380066, Training Accuracy= 0.88281 Iter 19200, Minibatch Loss= 0.340269, Training Accuracy= 0.91406 Iter 20480, Minibatch Loss= 0.351742, Training Accuracy= 0.88281 Iter 21760, Minibatch Loss= 0.278537, Training Accuracy= 0.91406 Iter 23040, Minibatch Loss= 0.345716, Training Accuracy= 0.89844 Iter 24320, Minibatch Loss= 0.338512, Training Accuracy= 0.92969 Iter 25600, Minibatch Loss= 0.301185, Training Accuracy= 0.92188 Iter 26880, Minibatch Loss= 0.335147, Training Accuracy= 0.90625 Iter 28160, Minibatch Loss= 0.329022, Training Accuracy= 0.89062 Iter 29440, Minibatch Loss= 0.278270, Training Accuracy= 0.90625 Iter 30720, Minibatch Loss= 0.141217, Training Accuracy= 0.96094 Iter 32000, Minibatch Loss= 0.316236, Training Accuracy= 0.91406 Iter 33280, Minibatch Loss= 0.170242, Training Accuracy= 0.96094 Iter 34560, Minibatch Loss= 0.169931, Training Accuracy= 0.96094 Iter 35840, Minibatch Loss= 0.261950, Training Accuracy= 0.91406 Iter 37120, Minibatch Loss= 0.275674, Training Accuracy= 0.92188 Iter 38400, Minibatch Loss= 0.269527, Training Accuracy= 0.92188 Iter 39680, Minibatch Loss= 0.288827, Training Accuracy= 0.92969 Iter 40960, Minibatch Loss= 0.123902, Training Accuracy= 0.97656 Iter 42240, Minibatch Loss= 0.222748, Training Accuracy= 0.93750 Iter 43520, Minibatch Loss= 0.275202, Training Accuracy= 0.91406 Iter 44800, Minibatch Loss= 0.114382, Training Accuracy= 0.97656 Iter 46080, Minibatch Loss= 0.232849, Training Accuracy= 0.93750 Iter 47360, Minibatch Loss= 0.183887, Training Accuracy= 0.96094 Iter 48640, Minibatch Loss= 0.143083, Training Accuracy= 0.94531 Iter 49920, Minibatch Loss= 0.119365, Training Accuracy= 0.95312 Iter 51200, Minibatch Loss= 0.165366, Training Accuracy= 0.93750 Iter 52480, Minibatch Loss= 0.142011, Training Accuracy= 0.96875 Iter 53760, Minibatch Loss= 0.124913, Training Accuracy= 0.96875 Iter 55040, Minibatch Loss= 0.271773, Training Accuracy= 0.91406 Iter 56320, Minibatch Loss= 0.151414, Training Accuracy= 0.94531 Iter 57600, Minibatch Loss= 0.312587, Training Accuracy= 0.89844 Iter 58880, Minibatch Loss= 0.128200, Training Accuracy= 0.96094 Iter 60160, Minibatch Loss= 0.082254, Training Accuracy= 0.96875 Iter 61440, Minibatch Loss= 0.143054, Training Accuracy= 0.94531 Iter 62720, Minibatch Loss= 0.189484, Training Accuracy= 0.94531 Iter 64000, Minibatch Loss= 0.145638, Training Accuracy= 0.95312 Iter 65280, Minibatch Loss= 0.095384, Training Accuracy= 0.96875 Iter 66560, Minibatch Loss= 0.107284, Training Accuracy= 0.96094 Iter 67840, Minibatch Loss= 0.072667, Training Accuracy= 0.98438 Iter 69120, Minibatch Loss= 0.078070, Training Accuracy= 0.98438 Iter 70400, Minibatch Loss= 0.145665, Training Accuracy= 0.96875 Iter 71680, Minibatch Loss= 0.131772, Training Accuracy= 0.96094 Iter 72960, Minibatch Loss= 0.054384, Training Accuracy= 0.97656 Iter 74240, Minibatch Loss= 0.080717, Training Accuracy= 0.99219 Iter 75520, Minibatch Loss= 0.127881, Training Accuracy= 0.94531 Iter 76800, Minibatch Loss= 0.050177, Training Accuracy= 0.98438 Iter 78080, Minibatch Loss= 0.204734, Training Accuracy= 0.92969 Iter 79360, Minibatch Loss= 0.134242, Training Accuracy= 0.95312 Iter 80640, Minibatch Loss= 0.102928, Training Accuracy= 0.96094 Iter 81920, Minibatch Loss= 0.151719, Training Accuracy= 0.96094 Iter 83200, Minibatch Loss= 0.063140, Training Accuracy= 0.98438 Iter 84480, Minibatch Loss= 0.044821, Training Accuracy= 0.99219 Iter 85760, Minibatch Loss= 0.042410, Training Accuracy= 0.99219 Iter 87040, Minibatch Loss= 0.043164, Training Accuracy= 1.00000 Iter 88320, Minibatch Loss= 0.095371, Training Accuracy= 0.97656 Iter 89600, Minibatch Loss= 0.107568, Training Accuracy= 0.96094 Iter 90880, Minibatch Loss= 0.097504, Training Accuracy= 0.96875 Iter 92160, Minibatch Loss= 0.065621, Training Accuracy= 0.98438 Iter 93440, Minibatch Loss= 0.089349, Training Accuracy= 0.97656 Iter 94720, Minibatch Loss= 0.203302, Training Accuracy= 0.93750 Iter 96000, Minibatch Loss= 0.076798, Training Accuracy= 0.98438 Iter 97280, Minibatch Loss= 0.087679, Training Accuracy= 0.97656 Iter 98560, Minibatch Loss= 0.081388, Training Accuracy= 0.96875 Iter 99840, Minibatch Loss= 0.062613, Training Accuracy= 0.98438 Finished! Testing Accuracy: 0.9765625
实例67:在GRUCell中实现LN
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
from tensorflow.python.ops.rnn_cell_impl import RNNCell
from tensorflow.python.ops.rnn_cell_impl import LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_ops
print(tf.__version__)
tf.reset_default_graph()
def ln(tensor, scope = None, epsilon = 1e-5):
""" Layer normalizes a 2D tensor along its second axis """
assert(len(tensor.get_shape()) == 2)
m, v = tf.nn.moments(tensor, [1], keep_dims=True)
if not isinstance(scope, str):
scope = ''
with tf.variable_scope(scope + 'layer_norm'):
scale = tf.get_variable('scale',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(1))
shift = tf.get_variable('shift',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(0))
LN_initial = (tensor - m) / tf.sqrt(v + epsilon)
return LN_initial * scale + shift
_BIAS_VARIABLE_NAME = "bias"
_WEIGHTS_VARIABLE_NAME = "kernel"
class LNGRUCell(LayerRNNCell):
"""Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
Args:
num_units: int, The number of units in the GRU cell.
activation: Nonlinearity to use. Default: `tanh`.
reuse: (optional) Python boolean describing whether to reuse variables
in an existing scope. If not `True`, and the existing scope already has
the given variables, an error is raised.
kernel_initializer: (optional) The initializer to use for the weight and
projection matrices.
bias_initializer: (optional) The initializer to use for the bias.
name: String, the name of the layer. Layers with the same name will
share weights, but to avoid mistakes we require reuse=True in such
cases.
"""
def __init__(self,
num_units,
activation=None,
reuse=None,
kernel_initializer=None,
bias_initializer=None,
name=None):
#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来
super(LNGRUCell, self).__init__(_reuse=reuse, name=name)
# Inputs must be 2-dimensional.
self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._activation = activation or math_ops.tanh
self._kernel_initializer = kernel_initializer
self._bias_initializer = bias_initializer
@property
def state_size(self):
return self._num_units
@property
def output_size(self):
return self._num_units
def build(self, inputs_shape):
if inputs_shape[1].value is None:
raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
% inputs_shape)
input_depth = inputs_shape[1].value
self._gate_kernel = self.add_variable(
"gates/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, 2 * self._num_units],
initializer=self._kernel_initializer)
self._gate_bias = self.add_variable(
"gates/%s" % _BIAS_VARIABLE_NAME,
shape=[2 * self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.constant_initializer(1.0, dtype=self.dtype)))
self._candidate_kernel = self.add_variable(
"candidate/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, self._num_units],
initializer=self._kernel_initializer)
self._candidate_bias = self.add_variable(
"candidate/%s" % _BIAS_VARIABLE_NAME,
shape=[self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.zeros_initializer(dtype=self.dtype)))
self.built = True
def call(self, inputs, state):
"""Gated recurrent unit (GRU) with nunits cells."""
gate_inputs = math_ops.matmul(
array_ops.concat([inputs, state], 1), self._gate_kernel)
gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)
value = math_ops.sigmoid(gate_inputs)
r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
r = ln(r, scope = 'r/')# 新添加
u = ln(u, scope = 'u/')# 新添加
r_state = r * state
candidate = math_ops.matmul(
array_ops.concat([inputs, r_state], 1), self._candidate_kernel)
candidate = nn_ops.bias_add(candidate, self._candidate_bias)
c = self._activation(candidate)
new_h = u * state + (1 - u) * c
return new_h, new_h
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
x1 = tf.unstack(x, n_steps, 1)
#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)
#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
gru = LNGRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)
#4 创建动态RNN
outputs,_ = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2])
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 128
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 12.231302, Training Accuracy= 0.08594 Iter 2560, Minibatch Loss= 11.580451, Training Accuracy= 0.08594 Iter 3840, Minibatch Loss= 12.903033, Training Accuracy= 0.06250 Iter 5120, Minibatch Loss= 10.890457, Training Accuracy= 0.12500 Iter 6400, Minibatch Loss= 9.708689, Training Accuracy= 0.12500 Iter 7680, Minibatch Loss= 11.344480, Training Accuracy= 0.07031 Iter 8960, Minibatch Loss= 21.113043, Training Accuracy= 0.11719 Iter 10240, Minibatch Loss= 16.354618, Training Accuracy= 0.12500 Iter 11520, Minibatch Loss= 221.128204, Training Accuracy= 0.15625 Iter 12800, Minibatch Loss= 51.676239, Training Accuracy= 0.14062 Iter 14080, Minibatch Loss= 51.621651, Training Accuracy= 0.16406 Iter 15360, Minibatch Loss= 32.979252, Training Accuracy= 0.13281 Iter 16640, Minibatch Loss= 11.387983, Training Accuracy= 0.16406 Iter 17920, Minibatch Loss= 106.375748, Training Accuracy= 0.13281 Iter 19200, Minibatch Loss= 162.647675, Training Accuracy= 0.07031 Iter 20480, Minibatch Loss= 70.351883, Training Accuracy= 0.03125 Iter 21760, Minibatch Loss= 55.518311, Training Accuracy= 0.12500 Iter 23040, Minibatch Loss= 31.811563, Training Accuracy= 0.10938 Iter 24320, Minibatch Loss= 42.223648, Training Accuracy= 0.11719 Iter 25600, Minibatch Loss= 278.500854, Training Accuracy= 0.08594 Iter 26880, Minibatch Loss= 237.631607, Training Accuracy= 0.10938 Iter 28160, Minibatch Loss= 105.564705, Training Accuracy= 0.14062 Iter 29440, Minibatch Loss= 56.833820, Training Accuracy= 0.08594 Iter 30720, Minibatch Loss= 22.568174, Training Accuracy= 0.10938 Iter 32000, Minibatch Loss= 11.444073, Training Accuracy= 0.10156 Iter 33280, Minibatch Loss= 5.311275, Training Accuracy= 0.11719 Iter 34560, Minibatch Loss= 3.783988, Training Accuracy= 0.10938 Iter 35840, Minibatch Loss= 5.724679, Training Accuracy= 0.10938 Iter 37120, Minibatch Loss= 13.915645, Training Accuracy= 0.06250 Iter 38400, Minibatch Loss= 7.799623, Training Accuracy= 0.06250 Iter 39680, Minibatch Loss= 4.735735, Training Accuracy= 0.10156 Iter 40960, Minibatch Loss= 3.492851, Training Accuracy= 0.14062 Iter 42240, Minibatch Loss= 2.750641, Training Accuracy= 0.12500 Iter 43520, Minibatch Loss= 3.131867, Training Accuracy= 0.03125 Iter 44800, Minibatch Loss= 12.454140, Training Accuracy= 0.14062 Iter 46080, Minibatch Loss= 34.983681, Training Accuracy= 0.11719 Iter 47360, Minibatch Loss= 30.050432, Training Accuracy= 0.11719 Iter 48640, Minibatch Loss= 89.305038, Training Accuracy= 0.14062 Iter 49920, Minibatch Loss= 717.047119, Training Accuracy= 0.04688 Iter 51200, Minibatch Loss= 593.221436, Training Accuracy= 0.07812 Iter 52480, Minibatch Loss= 552.920593, Training Accuracy= 0.07812 Iter 53760, Minibatch Loss= 298.492462, Training Accuracy= 0.10938 Iter 55040, Minibatch Loss= 293.587799, Training Accuracy= 0.12500 Iter 56320, Minibatch Loss= 90.123314, Training Accuracy= 0.08594 Iter 57600, Minibatch Loss= 131.756165, Training Accuracy= 0.11719 Iter 58880, Minibatch Loss= 90.683121, Training Accuracy= 0.10156 Iter 60160, Minibatch Loss= 56.682476, Training Accuracy= 0.09375 Iter 61440, Minibatch Loss= 64.690117, Training Accuracy= 0.07031 Iter 62720, Minibatch Loss= 73.732903, Training Accuracy= 0.12500 Iter 64000, Minibatch Loss= 39.817921, Training Accuracy= 0.18750 Iter 65280, Minibatch Loss= 26.479805, Training Accuracy= 0.03906 Iter 66560, Minibatch Loss= 28.650154, Training Accuracy= 0.03906 Iter 67840, Minibatch Loss= 32.062138, Training Accuracy= 0.07812 Iter 69120, Minibatch Loss= 24.480392, Training Accuracy= 0.10938 Iter 70400, Minibatch Loss= 18.083035, Training Accuracy= 0.14062 Iter 71680, Minibatch Loss= 33.831909, Training Accuracy= 0.08594 Iter 72960, Minibatch Loss= 31.264828, Training Accuracy= 0.17969 Iter 74240, Minibatch Loss= 24.374344, Training Accuracy= 0.12500 Iter 75520, Minibatch Loss= 32.843307, Training Accuracy= 0.09375 Iter 76800, Minibatch Loss= 20.493128, Training Accuracy= 0.05469 Iter 78080, Minibatch Loss= 21.611864, Training Accuracy= 0.16406 Iter 79360, Minibatch Loss= 25.270271, Training Accuracy= 0.17188 Iter 80640, Minibatch Loss= 25.224813, Training Accuracy= 0.14844 Iter 81920, Minibatch Loss= 27.151264, Training Accuracy= 0.08594 Iter 83200, Minibatch Loss= 24.466787, Training Accuracy= 0.06250 Iter 84480, Minibatch Loss= 26.366667, Training Accuracy= 0.21094 Iter 85760, Minibatch Loss= 19.625292, Training Accuracy= 0.18750 Iter 87040, Minibatch Loss= 18.934591, Training Accuracy= 0.17188 Iter 88320, Minibatch Loss= 32.218357, Training Accuracy= 0.10938 Iter 89600, Minibatch Loss= 46.415516, Training Accuracy= 0.12500 Iter 90880, Minibatch Loss= 30.490982, Training Accuracy= 0.11719 Iter 92160, Minibatch Loss= 30.738121, Training Accuracy= 0.16406 Iter 93440, Minibatch Loss= 25.382904, Training Accuracy= 0.10938 Iter 94720, Minibatch Loss= 21.190367, Training Accuracy= 0.11719 Iter 96000, Minibatch Loss= 32.771732, Training Accuracy= 0.10938 Iter 97280, Minibatch Loss= 19.819483, Training Accuracy= 0.11719 Iter 98560, Minibatch Loss= 13.052861, Training Accuracy= 0.15625 Iter 99840, Minibatch Loss= 21.311205, Training Accuracy= 0.15625 Finished! Testing Accuracy: 0.140625
Ln多GRu1
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
from tensorflow.python.ops.rnn_cell_impl import RNNCell
from tensorflow.python.ops.rnn_cell_impl import LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_ops
tf.reset_default_graph()
def ln(tensor, scope = None, epsilon = 1e-5):
""" Layer normalizes a 2D tensor along its second axis """
assert(len(tensor.get_shape()) == 2)
m, v = tf.nn.moments(tensor, [1], keep_dims=True)
if not isinstance(scope, str):
scope = ''
with tf.variable_scope(scope + 'layer_norm'):
scale = tf.get_variable('scale',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(1))
shift = tf.get_variable('shift',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(0))
LN_initial = (tensor - m) / tf.sqrt(v + epsilon)
return LN_initial * scale + shift
class LNGRUCell(LayerRNNCell):
"""Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
Args:
num_units: int, The number of units in the GRU cell.
activation: Nonlinearity to use. Default: `tanh`.
reuse: (optional) Python boolean describing whether to reuse variables
in an existing scope. If not `True`, and the existing scope already has
the given variables, an error is raised.
kernel_initializer: (optional) The initializer to use for the weight and
projection matrices.
bias_initializer: (optional) The initializer to use for the bias.
name: String, the name of the layer. Layers with the same name will
share weights, but to avoid mistakes we require reuse=True in such
cases.
"""
def __init__(self,
num_units,
activation=None,
reuse=None,
kernel_initializer=None,
bias_initializer=None,
name=None):
#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来
super(LNGRUCell, self).__init__(_reuse=reuse, name=name)
# Inputs must be 2-dimensional.
self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._activation = activation or math_ops.tanh
self._kernel_initializer = kernel_initializer
self._bias_initializer = bias_initializer
@property
def state_size(self):
return self._num_units
@property
def output_size(self):
return self._num_units
def build(self, inputs_shape):
if inputs_shape[1].value is None:
raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
% inputs_shape)
input_depth = inputs_shape[1].value
self._gate_kernel = self.add_variable(
"gates/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, 2 * self._num_units],
initializer=self._kernel_initializer)
self._gate_bias = self.add_variable(
"gates/%s" % _BIAS_VARIABLE_NAME,
shape=[2 * self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.constant_initializer(1.0, dtype=self.dtype)))
self._candidate_kernel = self.add_variable(
"candidate/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, self._num_units],
initializer=self._kernel_initializer)
self._candidate_bias = self.add_variable(
"candidate/%s" % _BIAS_VARIABLE_NAME,
shape=[self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.zeros_initializer(dtype=self.dtype)))
self.built = True
def call(self, inputs, state):
"""Gated recurrent unit (GRU) with nunits cells."""
gate_inputs = math_ops.matmul(
array_ops.concat([inputs, state], 1), self._gate_kernel)
gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)
value = math_ops.sigmoid(gate_inputs)
r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
r = ln(r, scope = 'r/')# 新添加
u = ln(u, scope = 'u/')# 新添加
r_state = r * state
candidate = math_ops.matmul(
array_ops.concat([inputs, r_state], 1), self._candidate_kernel)
candidate = nn_ops.bias_add(candidate, self._candidate_bias)
c = self._activation(candidate)
new_h = u * state + (1 - u) * c
return new_h, new_h
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
stacked_rnn = []
for i in range(3):
stacked_rnn.append(LNGRUCell(n_hidden))
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
outputs,states = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序,取最后一个时序outputs[-1]=(?,256)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 11.682723, Training Accuracy= 0.08594 Iter 2560, Minibatch Loss= 12.917633, Training Accuracy= 0.13281 Iter 3840, Minibatch Loss= 11.147453, Training Accuracy= 0.14062 Iter 5120, Minibatch Loss= 11.501393, Training Accuracy= 0.10938 Iter 6400, Minibatch Loss= 11.329225, Training Accuracy= 0.14062 Iter 7680, Minibatch Loss= 12.902426, Training Accuracy= 0.12500 Iter 8960, Minibatch Loss= 14.787603, Training Accuracy= 0.12500 Iter 10240, Minibatch Loss= 14.239416, Training Accuracy= 0.11719 Iter 11520, Minibatch Loss= 11.831277, Training Accuracy= 0.14062 Iter 12800, Minibatch Loss= 13.174420, Training Accuracy= 0.12500 Iter 14080, Minibatch Loss= 10.532808, Training Accuracy= 0.07812 Iter 15360, Minibatch Loss= 21.689716, Training Accuracy= 0.11719 Iter 16640, Minibatch Loss= 18.296116, Training Accuracy= 0.10938 Iter 17920, Minibatch Loss= 12.612600, Training Accuracy= 0.14844 Iter 19200, Minibatch Loss= 8.998665, Training Accuracy= 0.09375 Iter 20480, Minibatch Loss= 7.482079, Training Accuracy= 0.13281 Iter 21760, Minibatch Loss= 5.310290, Training Accuracy= 0.20312 Iter 23040, Minibatch Loss= 11.217176, Training Accuracy= 0.10156 Iter 24320, Minibatch Loss= 10.468469, Training Accuracy= 0.11719 Iter 25600, Minibatch Loss= 9.238225, Training Accuracy= 0.09375 Iter 26880, Minibatch Loss= 9.928499, Training Accuracy= 0.10156 Iter 28160, Minibatch Loss= 11.853710, Training Accuracy= 0.05469 Iter 29440, Minibatch Loss= 11.066288, Training Accuracy= 0.03906 Iter 30720, Minibatch Loss= 8.290665, Training Accuracy= 0.14844 Iter 32000, Minibatch Loss= 7.704072, Training Accuracy= 0.14844 Iter 33280, Minibatch Loss= 7.526510, Training Accuracy= 0.13281 Iter 34560, Minibatch Loss= 10.274215, Training Accuracy= 0.08594 Iter 35840, Minibatch Loss= 9.874534, Training Accuracy= 0.07812 Iter 37120, Minibatch Loss= 7.862291, Training Accuracy= 0.07812 Iter 38400, Minibatch Loss= 7.622520, Training Accuracy= 0.10156 Iter 39680, Minibatch Loss= 5.845595, Training Accuracy= 0.13281 Iter 40960, Minibatch Loss= 4.240401, Training Accuracy= 0.14062 Iter 42240, Minibatch Loss= 3.776126, Training Accuracy= 0.11719 Iter 43520, Minibatch Loss= 4.534883, Training Accuracy= 0.08594 Iter 44800, Minibatch Loss= 4.593569, Training Accuracy= 0.08594 Iter 46080, Minibatch Loss= 4.776147, Training Accuracy= 0.06250 Iter 47360, Minibatch Loss= 4.990581, Training Accuracy= 0.07812 Iter 48640, Minibatch Loss= 4.928415, Training Accuracy= 0.08594 Iter 49920, Minibatch Loss= 4.748688, Training Accuracy= 0.09375 Iter 51200, Minibatch Loss= 2.824999, Training Accuracy= 0.14062 Iter 52480, Minibatch Loss= 2.998337, Training Accuracy= 0.15625 Iter 53760, Minibatch Loss= 3.980899, Training Accuracy= 0.12500 Iter 55040, Minibatch Loss= 3.022208, Training Accuracy= 0.17969 Iter 56320, Minibatch Loss= 3.015280, Training Accuracy= 0.13281 Iter 57600, Minibatch Loss= 2.676783, Training Accuracy= 0.13281 Iter 58880, Minibatch Loss= 6.211295, Training Accuracy= 0.10938 Iter 60160, Minibatch Loss= 4.801533, Training Accuracy= 0.14062 Iter 61440, Minibatch Loss= 7.826266, Training Accuracy= 0.08594 Iter 62720, Minibatch Loss= 4.575137, Training Accuracy= 0.08594 Iter 64000, Minibatch Loss= 4.202699, Training Accuracy= 0.11719 Iter 65280, Minibatch Loss= 5.797447, Training Accuracy= 0.04688 Iter 66560, Minibatch Loss= 3.881121, Training Accuracy= 0.12500 Iter 67840, Minibatch Loss= 3.441604, Training Accuracy= 0.11719 Iter 69120, Minibatch Loss= 3.097083, Training Accuracy= 0.10938 Iter 70400, Minibatch Loss= 3.482257, Training Accuracy= 0.09375 Iter 71680, Minibatch Loss= 3.060152, Training Accuracy= 0.09375 Iter 72960, Minibatch Loss= 2.767920, Training Accuracy= 0.14062 Iter 74240, Minibatch Loss= 2.690722, Training Accuracy= 0.10156 Iter 75520, Minibatch Loss= 2.394173, Training Accuracy= 0.08594 Iter 76800, Minibatch Loss= 2.383599, Training Accuracy= 0.11719 Iter 78080, Minibatch Loss= 1.973354, Training Accuracy= 0.23438 Iter 79360, Minibatch Loss= 2.104026, Training Accuracy= 0.23438 Iter 80640, Minibatch Loss= 2.073925, Training Accuracy= 0.21094 Iter 81920, Minibatch Loss= 1.942348, Training Accuracy= 0.25781 Iter 83200, Minibatch Loss= 2.008878, Training Accuracy= 0.26562 Iter 84480, Minibatch Loss= 1.999240, Training Accuracy= 0.28125 Iter 85760, Minibatch Loss= 1.956447, Training Accuracy= 0.23438 Iter 87040, Minibatch Loss= 1.900973, Training Accuracy= 0.25781 Iter 88320, Minibatch Loss= 2.097849, Training Accuracy= 0.25000 Iter 89600, Minibatch Loss= 2.013985, Training Accuracy= 0.28906 Iter 90880, Minibatch Loss= 1.828421, Training Accuracy= 0.28125 Iter 92160, Minibatch Loss= 1.944018, Training Accuracy= 0.29688 Iter 93440, Minibatch Loss= 1.839489, Training Accuracy= 0.33594 Iter 94720, Minibatch Loss= 1.945905, Training Accuracy= 0.28125 Iter 96000, Minibatch Loss= 2.672914, Training Accuracy= 0.10156 Iter 97280, Minibatch Loss= 2.854921, Training Accuracy= 0.03906 Iter 98560, Minibatch Loss= 2.538456, Training Accuracy= 0.04688 Iter 99840, Minibatch Loss= 2.212051, Training Accuracy= 0.19531 Finished! Testing Accuracy: 0.24
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)
from tensorflow.python.ops.rnn_cell_impl import RNNCell
from tensorflow.python.ops.rnn_cell_impl import LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_ops
tf.reset_default_graph()
print(tf.__version__)
def ln(tensor, scope = None, epsilon = 1e-5):
""" Layer normalizes a 2D tensor along its second axis """
assert(len(tensor.get_shape()) == 2)
m, v = tf.nn.moments(tensor, [1], keep_dims=True)
if not isinstance(scope, str):
scope = ''
with tf.variable_scope(scope + 'layer_norm'):
scale = tf.get_variable('scale',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(1))
shift = tf.get_variable('shift',
shape=[tensor.get_shape()[1]],
initializer=tf.constant_initializer(0))
LN_initial = (tensor - m) / tf.sqrt(v + epsilon)
return LN_initial * scale + shift
#class LNGRUCell(RNNCell):
# """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078)."""
#
# def __init__(self, num_units, input_size=None, activation=tanh):
# if input_size is not None:
# print("%s: The input_size parameter is deprecated." % self)
# self._num_units = num_units
# self._activation = activation
#
# @property
# def state_size(self):
# return self._num_units
#
# @property
# def output_size(self):
# return self._num_units
#
# def __call__(self, inputs, state):
# """Gated recurrent unit (GRU) with nunits cells."""
# with vs.variable_scope("Gates"): # Reset gate and update gate.,reuse=True
# # We start with bias of 1.0 to not reset and not update.
# value =_linear([inputs, state], 2 * self._num_units, True, 1.0)
# r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
# r = ln(r, scope = 'r/')
# u = ln(u, scope = 'u/')
# r, u = sigmoid(r), sigmoid(u)
# with vs.variable_scope("Candidate"):
## with vs.variable_scope("Layer_Parameters"):
# Cand = _linear([inputs, r *state], self._num_units, True)
# c_pre = ln(Cand, scope = 'new_h/')
# c = self._activation(c_pre)
# new_h = u * state + (1 - u) * c
# return new_h, new_h
class LNGRUCell(LayerRNNCell):
"""
Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
Args:
num_units: int, The number of units in the GRU cell.
activation: Nonlinearity to use. Default: `tanh`.
reuse: (optional) Python boolean describing whether to reuse variables
in an existing scope. If not `True`, and the existing scope already has
the given variables, an error is raised.
kernel_initializer: (optional) The initializer to use for the weight and
projection matrices.
bias_initializer: (optional) The initializer to use for the bias.
name: String, the name of the layer. Layers with the same name will
share weights, but to avoid mistakes we require reuse=True in such
cases.
"""
def __init__(self,
num_units,
activation=None,
reuse=None,
kernel_initializer=None,
bias_initializer=None,
name=None):
#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来
super(LNGRUCell, self).__init__(_reuse=reuse, name=name)
# Inputs must be 2-dimensional.
self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._activation = activation or math_ops.tanh
self._kernel_initializer = kernel_initializer
self._bias_initializer = bias_initializer
@property
def state_size(self):
return self._num_units
@property
def output_size(self):
return self._num_units
def build(self, inputs_shape):
if inputs_shape[1].value is None:
raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
% inputs_shape)
input_depth = inputs_shape[1].value
self._gate_kernel = self.add_variable(
"gates/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, 2 * self._num_units],
initializer=self._kernel_initializer)
self._gate_bias = self.add_variable(
"gates/%s" % _BIAS_VARIABLE_NAME,
shape=[2 * self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.constant_initializer(1.0, dtype=self.dtype)))
self._candidate_kernel = self.add_variable(
"candidate/%s" % _WEIGHTS_VARIABLE_NAME,
shape=[input_depth + self._num_units, self._num_units],
initializer=self._kernel_initializer)
self._candidate_bias = self.add_variable(
"candidate/%s" % _BIAS_VARIABLE_NAME,
shape=[self._num_units],
initializer=(
self._bias_initializer
if self._bias_initializer is not None
else init_ops.zeros_initializer(dtype=self.dtype)))
self.built = True
def call(self, inputs, state):
"""Gated recurrent unit (GRU) with nunits cells."""
gate_inputs = math_ops.matmul(
array_ops.concat([inputs, state], 1), self._gate_kernel)
gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)
value = math_ops.sigmoid(gate_inputs)
r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
r = ln(r, scope = 'r/')# 新添加
u = ln(u, scope = 'u/')# 新添加
r_state = r * state
candidate = math_ops.matmul(
array_ops.concat([inputs, r_state], 1), self._candidate_kernel)
candidate = nn_ops.bias_add(candidate, self._candidate_bias)
c = self._activation(candidate)
new_h = u * state + (1 - u) * c
return new_h, new_h
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10 # MNIST 列别 (0-9 ,一共10类)
batch_size = 128
tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])
stacked_rnn = []
for i in range(3):
stacked_rnn.append(LNGRUCell(n_hidden))
#stacked_rnn.append(tf.contrib.rnn.GRUCell(n_hidden))
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
#outputs,states = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
#outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序,取最后一个时序outputs[-1]=(?,256)
#pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
#
learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
training_iters = 100000
display_step = 10
# 启动session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
step = 1
# Keep training until reach max iterations
while step * batch_size < training_iters:
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Reshape data to get 28 seq of 28 elements
batch_x = batch_x.reshape((batch_size, n_steps, n_input))
# Run optimization op (backprop)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
if step % display_step == 0:
# 计算批次数据的准确率
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
print (" Finished!")
# 计算准确率 for 128 mnist test images
test_len = 100
test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
test_label = mnist.test.labels[:test_len]
print ("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
Iter 1280, Minibatch Loss= 12.811991, Training Accuracy= 0.09375 Iter 2560, Minibatch Loss= 12.545067, Training Accuracy= 0.07031 Iter 3840, Minibatch Loss= 11.355438, Training Accuracy= 0.07031 Iter 5120, Minibatch Loss= 10.871563, Training Accuracy= 0.10156 Iter 6400, Minibatch Loss= 11.371557, Training Accuracy= 0.11719 Iter 7680, Minibatch Loss= 11.038004, Training Accuracy= 0.10938 Iter 8960, Minibatch Loss= 11.118206, Training Accuracy= 0.07812 Iter 10240, Minibatch Loss= 10.915785, Training Accuracy= 0.10156 Iter 11520, Minibatch Loss= 10.551557, Training Accuracy= 0.17969 Iter 12800, Minibatch Loss= 17.443115, Training Accuracy= 0.07031 Iter 14080, Minibatch Loss= 9.930510, Training Accuracy= 0.16406 Iter 15360, Minibatch Loss= 10.727852, Training Accuracy= 0.08594 Iter 16640, Minibatch Loss= 14.488325, Training Accuracy= 0.10938 Iter 17920, Minibatch Loss= 15.069497, Training Accuracy= 0.10938 Iter 19200, Minibatch Loss= 9.160069, Training Accuracy= 0.10938 Iter 20480, Minibatch Loss= 11.217492, Training Accuracy= 0.06250 Iter 21760, Minibatch Loss= 9.194048, Training Accuracy= 0.07812 Iter 23040, Minibatch Loss= 6.942473, Training Accuracy= 0.06250 Iter 24320, Minibatch Loss= 7.054324, Training Accuracy= 0.10938 Iter 25600, Minibatch Loss= 7.900481, Training Accuracy= 0.06250 Iter 26880, Minibatch Loss= 7.393861, Training Accuracy= 0.14844 Iter 28160, Minibatch Loss= 6.575096, Training Accuracy= 0.12500 Iter 29440, Minibatch Loss= 6.346336, Training Accuracy= 0.08594 Iter 30720, Minibatch Loss= 6.810472, Training Accuracy= 0.11719 Iter 32000, Minibatch Loss= 6.466897, Training Accuracy= 0.10938 Iter 33280, Minibatch Loss= 5.635717, Training Accuracy= 0.11719 Iter 34560, Minibatch Loss= 4.871469, Training Accuracy= 0.07812 Iter 35840, Minibatch Loss= 4.517097, Training Accuracy= 0.12500 Iter 37120, Minibatch Loss= 4.759137, Training Accuracy= 0.13281 Iter 38400, Minibatch Loss= 4.027944, Training Accuracy= 0.13281 Iter 39680, Minibatch Loss= 4.569145, Training Accuracy= 0.10156 Iter 40960, Minibatch Loss= 2.790155, Training Accuracy= 0.13281 Iter 42240, Minibatch Loss= 3.320386, Training Accuracy= 0.14844 Iter 43520, Minibatch Loss= 3.730759, Training Accuracy= 0.11719 Iter 44800, Minibatch Loss= 6.588100, Training Accuracy= 0.10938 Iter 46080, Minibatch Loss= 3.605313, Training Accuracy= 0.06250 Iter 47360, Minibatch Loss= 3.313602, Training Accuracy= 0.08594 Iter 48640, Minibatch Loss= 2.854012, Training Accuracy= 0.12500 Iter 49920, Minibatch Loss= 3.053122, Training Accuracy= 0.09375 Iter 51200, Minibatch Loss= 3.263433, Training Accuracy= 0.07812 Iter 52480, Minibatch Loss= 3.164253, Training Accuracy= 0.10156 Iter 53760, Minibatch Loss= 2.568920, Training Accuracy= 0.08594 Iter 55040, Minibatch Loss= 2.622833, Training Accuracy= 0.08594 Iter 56320, Minibatch Loss= 2.574237, Training Accuracy= 0.10156 Iter 57600, Minibatch Loss= 2.320722, Training Accuracy= 0.14062 Iter 58880, Minibatch Loss= 2.416647, Training Accuracy= 0.14062 Iter 60160, Minibatch Loss= 2.378407, Training Accuracy= 0.12500 Iter 61440, Minibatch Loss= 2.236217, Training Accuracy= 0.21094 Iter 62720, Minibatch Loss= 2.288638, Training Accuracy= 0.15625 Iter 64000, Minibatch Loss= 2.273193, Training Accuracy= 0.16406 Iter 65280, Minibatch Loss= 2.243811, Training Accuracy= 0.16406 Iter 66560, Minibatch Loss= 2.228886, Training Accuracy= 0.21875 Iter 67840, Minibatch Loss= 2.233191, Training Accuracy= 0.19531 Iter 69120, Minibatch Loss= 2.175747, Training Accuracy= 0.21875 Iter 70400, Minibatch Loss= 2.162902, Training Accuracy= 0.23438 Iter 71680, Minibatch Loss= 2.325285, Training Accuracy= 0.14062 Iter 72960, Minibatch Loss= 2.281687, Training Accuracy= 0.19531 Iter 74240, Minibatch Loss= 2.340919, Training Accuracy= 0.14062 Iter 75520, Minibatch Loss= 2.432267, Training Accuracy= 0.12500 Iter 76800, Minibatch Loss= 2.398518, Training Accuracy= 0.13281 Iter 78080, Minibatch Loss= 2.573318, Training Accuracy= 0.10938 Iter 79360, Minibatch Loss= 2.431727, Training Accuracy= 0.15625 Iter 80640, Minibatch Loss= 2.496175, Training Accuracy= 0.10156 Iter 81920, Minibatch Loss= 2.485367, Training Accuracy= 0.10156 Iter 83200, Minibatch Loss= 2.326829, Training Accuracy= 0.09375 Iter 84480, Minibatch Loss= 2.413650, Training Accuracy= 0.14062 Iter 85760, Minibatch Loss= 2.327744, Training Accuracy= 0.13281 Iter 87040, Minibatch Loss= 2.327587, Training Accuracy= 0.13281 Iter 88320, Minibatch Loss= 2.335200, Training Accuracy= 0.11719 Iter 89600, Minibatch Loss= 2.370470, Training Accuracy= 0.15625 Iter 90880, Minibatch Loss= 2.429362, Training Accuracy= 0.09375 Iter 92160, Minibatch Loss= 2.286218, Training Accuracy= 0.12500 Iter 93440, Minibatch Loss= 2.334179, Training Accuracy= 0.14844 Iter 94720, Minibatch Loss= 2.313130, Training Accuracy= 0.15625 Iter 96000, Minibatch Loss= 2.388861, Training Accuracy= 0.10938 Iter 97280, Minibatch Loss= 2.468186, Training Accuracy= 0.16406 Iter 98560, Minibatch Loss= 2.380651, Training Accuracy= 0.12500 Iter 99840, Minibatch Loss= 2.311615, Training Accuracy= 0.08594 Finished! Testing Accuracy: 0.14
实例68:利用BiRNN实现语音识别
# -*- coding: utf-8 -*-
import numpy as np
import time
import tensorflow as tf
from tensorflow.python.ops import ctc_ops
from collections import Counter
## 自定义
yuyinutils = __import__("9-24__yuyinutils")
sparse_tuple_to_texts_ch = yuyinutils.sparse_tuple_to_texts_ch
ndarray_to_text_ch = yuyinutils.ndarray_to_text_ch
get_audio_and_transcriptch = yuyinutils.get_audio_and_transcriptch
pad_sequences = yuyinutils.pad_sequences
sparse_tuple_from = yuyinutils.sparse_tuple_from
get_wavs_lables = yuyinutils.get_wavs_lables
tf.reset_default_graph()
b_stddev = 0.046875
h_stddev = 0.046875
n_hidden = 1024
n_hidden_1 = 1024
n_hidden_2 =1024
n_hidden_5 = 1024
n_cell_dim = 1024
n_hidden_3 = 2 * 1024
keep_dropout_rate=0.95
relu_clip = 20
def BiRNN_model( batch_x, seq_length, n_input, n_context,n_character ,keep_dropout):
# batch_x_shape: [batch_size, n_steps, n_input + 2*n_input*n_context]
batch_x_shape = tf.shape(batch_x)
# 将输入转成时间序列优先
batch_x = tf.transpose(batch_x, [1, 0, 2])
# 再转成2维传入第一层
batch_x = tf.reshape(batch_x,
[-1, n_input + 2 * n_input * n_context]) # (n_steps*batch_size, n_input + 2*n_input*n_context)
# 使用clipped RELU activation and dropout.
# 1st layer
with tf.name_scope('fc1'):
b1 = variable_on_cpu('b1', [n_hidden_1], tf.random_normal_initializer(stddev=b_stddev))
h1 = variable_on_cpu('h1', [n_input + 2 * n_input * n_context, n_hidden_1],
tf.random_normal_initializer(stddev=h_stddev))
layer_1 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), relu_clip)
layer_1 = tf.nn.dropout(layer_1, keep_dropout)
# 2nd layer
with tf.name_scope('fc2'):
b2 = variable_on_cpu('b2', [n_hidden_2], tf.random_normal_initializer(stddev=b_stddev))
h2 = variable_on_cpu('h2', [n_hidden_1, n_hidden_2], tf.random_normal_initializer(stddev=h_stddev))
layer_2 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), relu_clip)
layer_2 = tf.nn.dropout(layer_2, keep_dropout)
# 3rd layer
with tf.name_scope('fc3'):
b3 = variable_on_cpu('b3', [n_hidden_3], tf.random_normal_initializer(stddev=b_stddev))
h3 = variable_on_cpu('h3', [n_hidden_2, n_hidden_3], tf.random_normal_initializer(stddev=h_stddev))
layer_3 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), relu_clip)
layer_3 = tf.nn.dropout(layer_3, keep_dropout)
# 双向rnn
with tf.name_scope('lstm'):
# Forward direction cell:
lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)
lstm_fw_cell = tf.contrib.rnn.DropoutWrapper(lstm_fw_cell,
input_keep_prob=keep_dropout)
# Backward direction cell:
lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)
lstm_bw_cell = tf.contrib.rnn.DropoutWrapper(lstm_bw_cell,
input_keep_prob=keep_dropout)
# `layer_3` `[n_steps, batch_size, 2*n_cell_dim]`
layer_3 = tf.reshape(layer_3, [-1, batch_x_shape[0], n_hidden_3])
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
cell_bw=lstm_bw_cell,
inputs=layer_3,
dtype=tf.float32,
time_major=True,
sequence_length=seq_length)
# 连接正反向结果[n_steps, batch_size, 2*n_cell_dim]
outputs = tf.concat(outputs, 2)
# to a single tensor of shape [n_steps*batch_size, 2*n_cell_dim]
outputs = tf.reshape(outputs, [-1, 2 * n_cell_dim])
with tf.name_scope('fc5'):
b5 = variable_on_cpu('b5', [n_hidden_5], tf.random_normal_initializer(stddev=b_stddev))
h5 = variable_on_cpu('h5', [(2 * n_cell_dim), n_hidden_5], tf.random_normal_initializer(stddev=h_stddev))
layer_5 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(outputs, h5), b5)), relu_clip)
layer_5 = tf.nn.dropout(layer_5, keep_dropout)
with tf.name_scope('fc6'):
# 全连接层用于softmax分类
b6 = variable_on_cpu('b6', [n_character], tf.random_normal_initializer(stddev=b_stddev))
h6 = variable_on_cpu('h6', [n_hidden_5, n_character], tf.random_normal_initializer(stddev=h_stddev))
layer_6 = tf.add(tf.matmul(layer_5, h6), b6)
# 将2维[n_steps*batch_size, n_character]转成3维 time-major [n_steps, batch_size, n_character].
layer_6 = tf.reshape(layer_6, [-1, batch_x_shape[0], n_character])
# Output shape: [n_steps, batch_size, n_character]
return layer_6
"""
used to create a variable in CPU memory.
"""
def variable_on_cpu(name, shape, initializer):
# Use the /cpu:0 device for scoped operations
with tf.device('/cpu:0'):
# Create or get apropos variable
var = tf.get_variable(name=name, shape=shape, initializer=initializer)
return var
wav_path='H:/tensorflow_projects/chap9/data_thchs30/wav/train'
label_file='H:/tensorflow_projects/chap9/data_thchs30/doc/trans/train.word.txt'
wav_files, labels = get_wavs_lables(wav_path,label_file)
print(wav_files[0], labels[0])
# wav/train/A11/A11_0.WAV -> 绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然
print("wav:",len(wav_files),"label",len(labels))
# 字表
all_words = []
for label in labels:
#print(label)
all_words += [word for word in label]
counter = Counter(all_words)
words = sorted(counter)
words_size= len(words)
word_num_map = dict(zip(words, range(words_size)))
print('字表大小:', words_size)
n_input = 26#计算美尔倒谱系数的个数
n_context = 9#对于每个时间点,要包含上下文样本的个数
batch_size =8
def next_batch(labels, start_idx = 0,batch_size=1,wav_files = wav_files):
filesize = len(labels)
end_idx = min(filesize, start_idx + batch_size)
idx_list = range(start_idx, end_idx)
txt_labels = [labels[i] for i in idx_list]
wav_files = [wav_files[i] for i in idx_list]
(source, audio_len, target, transcript_len) = get_audio_and_transcriptch(None,
wav_files,
n_input,
n_context,word_num_map,txt_labels)
start_idx += batch_size
# Verify that the start_idx is not larger than total available sample size
if start_idx >= filesize:
start_idx = -1
# Pad input to max_time_step of this batch
source, source_lengths = pad_sequences(source)#如果多个文件将长度统一,支持按最大截断或补0
sparse_labels = sparse_tuple_from(target)
return start_idx,source, source_lengths, sparse_labels
next_idx,source,source_len,sparse_lab = next_batch(labels,0,batch_size)
print(len(sparse_lab))
print(np.shape(source))
#print(sparse_lab)
t = sparse_tuple_to_texts_ch(sparse_lab,words)
print(t[0])
#source已经将变为前9(不够补空)+本身+后9,每个26,第一个顺序是第10个的数据。
# shape = [batch_size, max_stepsize, n_input + (2 * n_input * n_context)]
# the batch_size and max_stepsize每步都是变长的。
input_tensor = tf.placeholder(tf.float32, [None, None, n_input + (2 * n_input * n_context)], name='input')#语音log filter bank or MFCC features
# Use sparse_placeholder; will generate a SparseTensor, required by ctc_loss op.
targets = tf.sparse_placeholder(tf.int32, name='targets')#文本
# 1d array of size [batch_size]
seq_length = tf.placeholder(tf.int32, [None], name='seq_length')#序列长
keep_dropout= tf.placeholder(tf.float32)
# logits is the non-normalized output/activations from the last layer.
# logits will be input for the loss function.
# nn_model is from the import statement in the load_model function
logits = BiRNN_model( input_tensor, tf.to_int64(seq_length), n_input, n_context,words_size +1,keep_dropout)
#调用ctc loss
avg_loss = tf.reduce_mean(ctc_ops.ctc_loss(targets, logits, seq_length))
#[optimizer]
learning_rate = 0.001
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(avg_loss)
with tf.name_scope("decode"):
decoded, log_prob = ctc_ops.ctc_beam_search_decoder( logits, seq_length, merge_repeated=False)
with tf.name_scope("accuracy"):
distance = tf.edit_distance( tf.cast(decoded[0], tf.int32), targets)
# 计算label error rate (accuracy)
ler = tf.reduce_mean(distance, name='label_error_rate')
epochs = 100
savedir = "log/yuyinchalltest/"
saver = tf.train.Saver(max_to_keep=1) # 生成saver
# create the session
sess = tf.Session()
# 没有模型的话,就重新初始化
sess.run(tf.global_variables_initializer())
kpt = tf.train.latest_checkpoint(savedir)
print("kpt:",kpt)
startepo= 0
if kpt!=None:
saver.restore(sess, kpt)
ind = kpt.find("-")
startepo = int(kpt[ind+1:])
print(startepo)
# 准备运行训练步骤
section = '\n{0:=^40}\n'
print(section.format('Run training epoch'))
train_start = time.time()
for epoch in range(epochs):#样本集迭代次数
epoch_start = time.time()
if epoch<startepo:
continue
print("epoch start:",epoch,"total epochs= ",epochs)
#######################run batch####
n_batches_per_epoch = int(np.ceil(len(labels) / batch_size))
print("total loop ",n_batches_per_epoch,"in one epoch,",batch_size,"items in one loop")
train_cost = 0
train_ler = 0
next_idx =0
for batch in range(n_batches_per_epoch):#一次batch_size,取多少次
#取数据
next_idx,source,source_lengths,sparse_labels = \
next_batch(labels,next_idx ,batch_size)
feed = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:keep_dropout_rate}
#计算 avg_loss optimizer ;
batch_cost, _ = sess.run([avg_loss, optimizer], feed_dict=feed )
train_cost += batch_cost
if (batch +1)%20 == 0:
print('loop:',batch, 'Train cost: ', train_cost/(batch+1))
feed2 = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:1.0}
d,train_ler = sess.run([decoded[0],ler], feed_dict=feed2)
dense_decoded = tf.sparse_tensor_to_dense( d, default_value=-1).eval(session=sess)
dense_labels = sparse_tuple_to_texts_ch(sparse_labels,words)
counter =0
print('Label err rate: ', train_ler)
for orig, decoded_arr in zip(dense_labels, dense_decoded):
# convert to strings
decoded_str = ndarray_to_text_ch(decoded_arr,words)
print(' file {}'.format( counter))
print('Original: {}'.format(orig))
print('Decoded: {}'.format(decoded_str))
counter=counter+1
break
epoch_duration = time.time() - epoch_start
log = 'Epoch {}/{}, train_cost: {:.3f}, train_ler: {:.3f}, time: {:.2f} sec'
print(log.format(epoch ,epochs, train_cost,train_ler,epoch_duration))
saver.save(sess, savedir+"yuyinch.cpkt", global_step=epoch)
train_duration = time.time() - train_start
print('Training complete, total duration: {:.2f} min'.format(train_duration / 60))
sess.close()
H:/tensorflow_projects/chap9/data_thchs30/wav/train\A11\A11_0.WAV 绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然 wav: 8911 label 8911 字表大小: 2666 3 (8, 584, 494) 绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然 WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:197: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version. Instructions for updating: Use tf.cast instead. WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer. WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:53: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version. Instructions for updating: Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`. WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0. For more information, please see: * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md * https://github.com/tensorflow/addons If you depend on functionality not listed there, please file an issue. WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:73: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version. Instructions for updating: This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0. WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:89: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version. Instructions for updating: Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version. Instructions for updating: Please use `keras.layers.RNN(cell)`, which is equivalent to this API WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version. Instructions for updating: Use tf.cast instead. kpt: None ===========Run training epoch=========== epoch start: 0 total epochs= 100 total loop 1114 in one epoch, 8 items in one loop loop: 19 Train cost: 858.0828048706055 Label err rate: 0.98041475 file 0 Original: 小 鸳鸯 成长 很快 到了 深秋 在 北方 出生 的 小 鸳鸯 便 能 跟随 鸳鸯 大群 一起 南下 越冬 了 Decoded: loop: 39 Train cost: 575.0958625793457 Label err rate: 0.9796773 file 0 Original: 马 晓 年 作画 的 特征 与 神韵 其 画风 区别 于 文人 画 和 年画 可谓 雅俗共赏 Decoded: loop: 59 Train cost: 475.83005854288734 Label err rate: 0.97848856 file 0 Original: 另外 加工 修理 和 修配 业务 不 属于 营业税 的 应 税 劳务 不 缴纳 营业税 Decoded: loop: 79 Train cost: 422.05160999298096 Label err rate: 0.9786789 file 0 Original: 这 碗 离 娘 饭 姑娘 再有 离 娘 痛楚 也 要 每样 都 吃 一点 才 算 循 规 遵 俗 的 Decoded: loop: 99 Train cost: 389.91693817138673 Label err rate: 0.97975487 file 0 Original: 国防大学 政委 王茂 润 中将 与 延安 八一 敬老院 八 十五岁 的 老红军 柴 福耀 亲切 交谈 Decoded: loop: 119 Train cost: 368.8437096913656 Label err rate: 0.973868 file 0 Original: 仅 绘画 而论 齐白石 是 巍巍 昆仑 可 这位 附庸风雅 的 门外汉 连 一块 石头 都 不是 Decoded: 的 loop: 139 Train cost: 353.2190541948591 Label err rate: 0.95776415 file 0 Original: 这次 全国 青年 排球 联赛 共 设 天津 舟山 武汉 三个 赛区 每个 赛区 的 前 两 名将 参加 复赛 Decoded: 的 loop: 159 Train cost: 341.92204132080076 Label err rate: 0.9404359 file 0 Original: 人代会 开幕 那天 奥 申 委 执行主席 伍 绍 祖 在 人民大会堂 东 大厅 一 露面 就 被 团团 围住 Decoded: 的 龚龚 loop: 179 Train cost: 333.7818296644423 Label err rate: 0.9725291 file 0 Original: 主要原因 是 世界 红 白藤 主产 国 印尼 禁止 红 白藤 原料 出口 导致 国际 红 白藤 原料 价格 上涨 Decoded: 龚龚 loop: 199 Train cost: 326.5086625671387 Label err rate: 0.93530095 file 0 Original: 赛 距 为 职业 运动员 最长 二十四 千米 业余 运动员 最长 二十一 千米 青年 运动员 最长 十五 千米 Decoded: 的 龚龚 loop: 219 Train cost: 321.0388072620739 Label err rate: 0.980114 file 0 Original: 据说 母亲 生下 没 一年 功夫 染 上了 一 种 怪病 不吃 不喝 乱 哭 乱 闹 急 得 外婆 满山遍野 去找 草药 来 驱鬼 Decoded: loop: 239 Train cost: 316.15502440134685 Label err rate: 0.9510975 file 0 Original: 所谓 软 是指 软件 包括 专利 使用权 设计 图纸 技术 诀窍 纯 技术 引进 不 带 硬件 Decoded: 的 loop: 259 Train cost: 311.4005855266864 Label err rate: 0.9620612 file 0 Original: 要知道 秦俑 二 号 坑 的 军 阵 性质 首先 要 了解一下 学者 们 关于 秦俑 一号 坑 军 阵 性质 的 探讨 Decoded: 龚龚 loop: 279 Train cost: 307.48678610665456 Label err rate: 0.9382373 file 0 Original: 使用 这样的 牙刷 会 不同 程度 地 刺伤 牙龈 造成 牙龈出血 并 导致 牙齿 过度 磨损 Decoded: 的 龚龚 loop: 299 Train cost: 304.18842356363933 Label err rate: 0.9661161 file 0 Original: 新华社 基 多 二月 六 日电 最近 二 十天 来 厄瓜多尔 全国 疟疾 迅速 蔓延 已有 多 人 死亡 Decoded: 的 loop: 319 Train cost: 301.10915036201476
实例69:利用RNN训练语言模型
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import random
import time
from collections import Counter
start_time = time.time()
def elapsed(sec):
if sec<60:
return str(sec) + " sec"
elif sec<(60*60):
return str(sec/60) + " min"
else:
return str(sec/(60*60)) + " hr"
# Target log path
tf.reset_default_graph()
training_file = 'wordstest.txt'
#中文多文件
def readalltxt(txt_files):
labels = []
for txt_file in txt_files:
target = get_ch_lable(txt_file)
labels.append(target)
return labels
#中文字
def get_ch_lable(txt_file):
labels= ""
with open(txt_file, 'rb') as f:
for label in f:
#labels =label.decode('utf-8')
labels =labels+label.decode('gb2312')
return labels
#优先转文件里的字符到向量
def get_ch_lable_v(txt_file,word_num_map,txt_label=None):
words_size = len(word_num_map)
to_num = lambda word: word_num_map.get(word, words_size)
if txt_file!= None:
txt_label = get_ch_lable(txt_file)
labels_vector = list(map(to_num, txt_label))
return labels_vector
training_data =get_ch_lable(training_file)
print("Loaded training data...")
print(len(training_data))
counter = Counter(training_data)
words = sorted(counter)
words_size= len(words)
word_num_map = dict(zip(words, range(words_size)))
print('字表大小:', words_size)
wordlabel = get_ch_lable_v(training_file,word_num_map)
#参数设置
learning_rate = 0.001
training_iters = 10000
display_step = 1000
n_input = 4
n_hidden1 = 256
n_hidden2 = 512
n_hidden3 = 512
# tf Graph input
x = tf.placeholder("float", [None, n_input,1])
wordy = tf.placeholder("float", [None, words_size])
x1 = tf.reshape(x, [-1, n_input])
x2 = tf.split(x1,n_input,1)
# 2-layer LSTM, each layer has n_hidden units.
rnn_cell = rnn.MultiRNNCell([rnn.LSTMCell(n_hidden1),rnn.LSTMCell(n_hidden2),rnn.LSTMCell(n_hidden3)])
# generate prediction
outputs, states = rnn.static_rnn(rnn_cell, x2, dtype=tf.float32)
# last output
pred = tf.contrib.layers.fully_connected(outputs[-1],words_size,activation_fn = None)
# Loss optimizer
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=wordy))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)
# Model evaluation
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(wordy,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
savedir = "log/rnnword/"
saver = tf.train.Saver(max_to_keep=1) # 生成saver
# 启动session
with tf.Session() as session:
session.run(tf.global_variables_initializer())
step = 0
offset = random.randint(0,n_input+1)
end_offset = n_input + 1
acc_total = 0
loss_total = 0
kpt = tf.train.latest_checkpoint(savedir)
print("kpt:",kpt)
startepo= 0
if kpt!=None:
saver.restore(session, kpt)
ind = kpt.find("-")
startepo = int(kpt[ind+1:])
print(startepo)
step = startepo
while step < training_iters:
# 随机取一个位置偏移
if offset > (len(training_data)-end_offset):
offset = random.randint(0, n_input+1)
inwords = [ [wordlabel[ i]] for i in range(offset, offset+n_input) ]#按照指定的位置偏移获取后面的4个文字向量当作输入
inwords = np.reshape(np.array(inwords), [-1, n_input, 1])
out_onehot= np.zeros([words_size], dtype=float)
out_onehot[wordlabel[offset+n_input]] = 1.0
out_onehot = np.reshape(out_onehot,[1,-1])#所有的字都变成onehot
_, acc, lossval, onehot_pred = session.run([optimizer, accuracy, loss, pred],feed_dict={x: inwords, wordy: out_onehot})
loss_total += lossval
acc_total += acc
if (step+1) % display_step == 0:
print("Iter= " + str(step+1) + ", Average Loss= " + \
"{:.6f}".format(loss_total/display_step) + ", Average Accuracy= " + \
"{:.2f}%".format(100*acc_total/display_step))
acc_total = 0
loss_total = 0
in2 = [words [wordlabel[i]] for i in range(offset, offset + n_input)]
out2 = words [wordlabel[offset + n_input]]
out_pred=words[int(tf.argmax(onehot_pred, 1).eval())]
print("%s - [%s] vs [%s]" % (in2,out2,out_pred))
saver.save(session, savedir+"rnnwordtest.cpkt", global_step=step)
step += 1
offset += (n_input+1)#中间隔了一个,作为预测
print("Finished!")
saver.save(session, savedir+"rnnwordtest.cpkt", global_step=step)
print("Elapsed time: ", elapsed(time.time() - start_time))
while True:
prompt = "请输入%s个字: " % n_input
sentence = input(prompt)
inputword = sentence.strip()
if len(inputword) != n_input:
print("您输入的字符长度为:",len(inputword),"请输入4个字")
continue
try:
inputword = get_ch_lable_v(None,word_num_map,inputword)
for i in range(32):
keys = np.reshape(np.array(inputword), [-1, n_input, 1])
onehot_pred = session.run(pred, feed_dict={x: keys})
onehot_pred_index = int(tf.argmax(onehot_pred, 1).eval())
sentence = "%s%s" % (sentence,words[onehot_pred_index])
inputword = inputword[1:]
inputword.append(onehot_pred_index)
print(sentence)
except:
print("该字我还没学会")
实例70:word2vect
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
import random
import collections
from collections import Counter
import jieba
from sklearn.manifold import TSNE
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rcParams['font.sans-serif']=['SimHei']#用来正常显示中文标签
mpl.rcParams['font.family'] = 'STSong'
mpl.rcParams['font.size'] = 20
training_file = '人体阴阳与电能.txt'
#中文字
def get_ch_lable(txt_file):
labels= ""
with open(txt_file, 'rb') as f:
for label in f:
#labels =label.decode('utf-8')
labels =labels+label.decode('gb2312')
return labels
#分词
def fenci(training_data):
seg_list = jieba.cut(training_data) # 默认是精确模式
training_ci = " ".join(seg_list)
training_ci = training_ci.split()
#以空格将字符串分开
training_ci = np.array(training_ci)
training_ci = np.reshape(training_ci, [-1, ])
return training_ci
def build_dataset(words, n_words):
"""Process raw inputs into a dataset."""
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(n_words - 1))
dictionary = dict()
for word, _ in count:
dictionary[word] = len(dictionary)
data = list()
unk_count = 0
for word in words:
if word in dictionary:
index = dictionary[word]
else:
index = 0 # dictionary['UNK']
unk_count += 1
data.append(index)
count[0][1] = unk_count
reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
return data, count, dictionary, reversed_dictionary
training_data =get_ch_lable(training_file)
print("总字数",len(training_data))
training_ci =fenci(training_data)
#print(training_ci)
print("总词数",len(training_ci))
training_label, count, dictionary, words = build_dataset(training_ci, 350)
words_size = len(dictionary)
print("字典词数",words_size)
#print(training_label)#将文本转为词向量
#print(words)#每个编号对应的词
#print(dictionary)#每个词对应的编号
#print(count)#每个词对应的个数
####################################################
print('Sample data', training_label[:10], [words[i] for i in training_label[:10]])
data_index = 0
def generate_batch(data,batch_size, num_skips, skip_window):
global data_index
assert batch_size % num_skips == 0
assert num_skips <= 2 * skip_window
batch = np.ndarray(shape=(batch_size), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
span = 2 * skip_window + 1 # [ skip_window target skip_window ]
buffer = collections.deque(maxlen=span)
if data_index + span > len(data):
data_index = 0
buffer.extend(data[data_index:data_index + span])
data_index += span
for i in range(batch_size // num_skips):
target = skip_window # target label at the center of the buffer
targets_to_avoid = [skip_window]
for j in range(num_skips):
while target in targets_to_avoid:
target = random.randint(0, span - 1)
targets_to_avoid.append(target)
batch[i * num_skips + j] = buffer[skip_window]
labels[i * num_skips + j, 0] = buffer[target]
if data_index == len(data):
#print(data_index,len(data),span,len(data[:span]))
#buffer[:] = data[:span]
buffer = data[:span]
data_index = span
else:
buffer.append(data[data_index])
data_index += 1
# Backtrack a little bit to avoid skipping words in the end of a batch
data_index = (data_index + len(data) - span) % len(data)
return batch, labels
batch, labels = generate_batch(training_label,batch_size=8, num_skips=2, skip_window=1)
for i in range(8):# 取第一个字,后一个是标签,再取其前一个字当标签,
print(batch[i], words[batch[i]], '->', labels[i, 0], words[labels[i, 0]])
batch_size = 128
embedding_size = 128 # Dimension of the embedding vector.
skip_window = 1 # How many words to consider left and right.
num_skips = 2 # How many times to reuse an input to generate a label.
valid_size = 16 # Random set of words to evaluate similarity on.
valid_window =np.int32( words_size/2 ) # Only pick dev samples in the head of the distribution.
print("valid_window",valid_window)
valid_examples = np.random.choice(valid_window, valid_size, replace=False)#0-words_size/2,中的数取16个。不能重复。
num_sampled = 64 # Number of negative examples to sample.
tf.reset_default_graph()
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):
# Look up embeddings for inputs.
embeddings = tf.Variable(tf.random_uniform([words_size, embedding_size], -1.0, 1.0))#94个,每个128个向量
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
# Construct the variables for the NCE loss
nce_weights = tf.Variable( tf.truncated_normal([words_size, embedding_size],
stddev=1.0 / tf.sqrt(np.float32(embedding_size))))
nce_biases = tf.Variable(tf.zeros([words_size]))
# Compute the average NCE loss for the batch.
# tf.nce_loss automatically draws a new sample of the negative labels each
# time we evaluate the loss.
loss = tf.reduce_mean(
tf.nn.nce_loss(weights=nce_weights, biases=nce_biases,
labels=train_labels, inputs=embed,
num_sampled=num_sampled, num_classes=words_size))
# Construct the SGD optimizer using a learning rate of 1.0.
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
# Compute the cosine similarity between minibatch examples and all embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
print("________________________",similarity.shape)
#Begin training.
num_steps = 100001
with tf.Session() as sess:
sess.run( tf.global_variables_initializer() )
print('Initialized')
average_loss = 0
for step in range(num_steps):
batch_inputs, batch_labels = generate_batch(training_label, batch_size, num_skips, skip_window)
feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
# We perform one update step by evaluating the optimizer op (including it
# in the list of returned values for session.run()
_, loss_val = sess.run([optimizer, loss], feed_dict=feed_dict)
average_loss += loss_val
#通过打印测试可以看到 embed的值在逐渐的被调节
# emv = sess.run(embed,feed_dict = {train_inputs: [37,18]})
# print("emv-------------------",emv[0])
if step % 2000 == 0:
if step > 0:
average_loss /= 2000
# The average loss is an estimate of the loss over the last 2000 batches.
print('Average loss at step ', step, ': ', average_loss)
average_loss = 0
# Note that this is expensive (~20% slowdown if computed every 500 steps)
if step % 10000 == 0:
sim = similarity.eval(session=sess)
#print(valid_size)
for i in range(valid_size):
valid_word = words[valid_examples[i]]
#print("valid_word",valid_word)#16
top_k = 8 # number of nearest neighbors
nearest = (-sim[i, :]).argsort()[1:top_k + 1] #argsort函数返回的是数组值从小到大的索引值
#print("nearest",nearest,top_k)
log_str = 'Nearest to %s:' % valid_word
for k in range(top_k):
close_word = words[nearest[k]]
log_str = '%s,%s' % (log_str, close_word)
print(log_str)
final_embeddings = normalized_embeddings.eval()
def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
plt.figure(figsize=(18, 18)) # in inches
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
plt.scatter(x, y)
plt.annotate(label,xy=(x, y),xytext=(5, 2), textcoords='offset points',
ha='right',va='bottom')
plt.savefig(filename)
try:
# pylint: disable=g-import-not-at-top
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
plot_only = 80#输出100个词
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [words[i] for i in range(plot_only)]
#print(labels)
plot_with_labels(low_dim_embs, labels)
except ImportError:
print('Please install sklearn, matplotlib, and scipy to show embeddings.')
实例71:word2vect自定义候选采样
# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
import random
import collections
from collections import Counter
import jieba
from sklearn.manifold import TSNE
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rcParams['font.sans-serif']=['SimHei']#用来正常显示中文标签
mpl.rcParams['font.family'] = 'STSong'
mpl.rcParams['font.size'] = 20
training_file = '人体阴阳与电能.txt'
#中文字
def get_ch_lable(txt_file):
labels= ""
with open(txt_file, 'rb') as f:
for label in f:
#labels =label.decode('utf-8')
labels =labels+label.decode('gb2312')
return labels
#分词
def fenci(training_data):
seg_list = jieba.cut(training_data) # 默认是精确模式
training_ci = " ".join(seg_list)
training_ci = training_ci.split()
#以空格将字符串分开
training_ci = np.array(training_ci)
training_ci = np.reshape(training_ci, [-1, ])
return training_ci
def build_dataset(words, n_words):
"""Process raw inputs into a dataset."""
count = [['UNK', -1]]
count.extend(collections.Counter(words).most_common(n_words - 1))
dictionary = dict()
vocab_freqs = []
for word, nvocab in count:
dictionary[word] = len(dictionary)
vocab_freqs.append(nvocab)
data = list()
unk_count = 0
for word in words:
if word in dictionary:
index = dictionary[word]
else:
index = 0 # dictionary['UNK']
unk_count += 1
data.append(index)
count[0][1] = unk_count
reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
return data, count, dictionary, reversed_dictionary,vocab_freqs
training_data =get_ch_lable(training_file)
print("总字数",len(training_data))
training_ci =fenci(training_data)
#print(training_ci)
print("总词数",len(training_ci))
training_label, count, dictionary, words,vocab_freqs = build_dataset(training_ci, 350)
words_size = len(dictionary)
print("字典词数",words_size)
#print(training_label)#将文本转为词向量
#print(words)#每个编号对应的词
#print(dictionary)#每个词对应的编号
#print(count)#每个词对应的个数
####################################################
print('Sample data', training_label[:10], [words[i] for i in training_label[:10]])
data_index = 0
def generate_batch(data,batch_size, num_skips, skip_window):
global data_index
assert batch_size % num_skips == 0
assert num_skips <= 2 * skip_window
batch = np.ndarray(shape=(batch_size), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
span = 2 * skip_window + 1 # [ skip_window target skip_window ]
buffer = collections.deque(maxlen=span)
if data_index + span > len(data):
data_index = 0
buffer.extend(data[data_index:data_index + span])
data_index += span
for i in range(batch_size // num_skips):
target = skip_window # target label at the center of the buffer
targets_to_avoid = [skip_window]
for j in range(num_skips):
while target in targets_to_avoid:
target = random.randint(0, span - 1)
targets_to_avoid.append(target)
batch[i * num_skips + j] = buffer[skip_window]
labels[i * num_skips + j, 0] = buffer[target]
if data_index == len(data):
#print(data_index,len(data),span,len(data[:span]))
#buffer[:] = data[:span]
buffer = data[:span]
data_index = span
else:
buffer.append(data[data_index])
data_index += 1
# Backtrack a little bit to avoid skipping words in the end of a batch
data_index = (data_index + len(data) - span) % len(data)
return batch, labels
batch, labels = generate_batch(training_label,batch_size=8, num_skips=2, skip_window=1)
for i in range(8):# 取第一个字,后一个是标签,再取其前一个字当标签,
print(batch[i], words[batch[i]], '->', labels[i, 0], words[labels[i, 0]])
batch_size = 128
embedding_size = 128 # Dimension of the embedding vector.
skip_window = 1 # How many words to consider left and right.
num_skips = 2 # How many times to reuse an input to generate a label.
valid_size = 16 # Random set of words to evaluate similarity on.
valid_window =np.int32( words_size/2 ) # Only pick dev samples in the head of the distribution.
print("valid_window",valid_window)
valid_examples = np.random.choice(valid_window, valid_size, replace=False)#0-words_size/2,中的数取16个。不能重复。
num_sampled = 64 # Number of negative examples to sample.
tf.reset_default_graph()
train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):
# Look up embeddings for inputs.
embeddings = tf.Variable(tf.random_uniform([words_size, embedding_size], -1.0, 1.0))#94个,每个128个向量
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
# Construct the variables for the NCE loss
nce_weights = tf.Variable( tf.truncated_normal([words_size, embedding_size],
stddev=1.0 / tf.sqrt(np.float32(embedding_size))))
nce_biases = tf.Variable(tf.zeros([words_size]))
vocab_freqs[0] = 90
sampled = tf.nn.fixed_unigram_candidate_sampler(
true_classes=tf.cast(train_labels,tf.int64),
num_true=1,
num_sampled=num_sampled,
unique=True,
range_max=words_size,
unigrams=vocab_freqs)
loss = tf.reduce_mean(
tf.nn.sampled_softmax_loss(weights=nce_weights, biases=nce_biases,
labels=train_labels, inputs=embed,
num_sampled=num_sampled, num_classes=words_size,sampled_values=sampled))
# Construct the SGD optimizer using a learning rate of 1.0.
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
# Compute the cosine similarity between minibatch examples and all embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
print("________________________",similarity.shape)
#Begin training.
num_steps = 100001
with tf.Session() as sess:
sess.run( tf.global_variables_initializer() )
print('Initialized')
average_loss = 0
for step in range(num_steps):
batch_inputs, batch_labels = generate_batch(training_label, batch_size, num_skips, skip_window)
feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
# We perform one update step by evaluating the optimizer op (including it
# in the list of returned values for session.run()
_, loss_val = sess.run([optimizer, loss], feed_dict=feed_dict)
average_loss += loss_val
#通过打印测试可以看到 embed的值在逐渐的被调节
# emv = sess.run(embed,feed_dict = {train_inputs: [37,18]})
# print("emv-------------------",emv[0])
if step % 2000 == 0:
if step > 0:
average_loss /= 2000
# The average loss is an estimate of the loss over the last 2000 batches.
print('Average loss at step ', step, ': ', average_loss)
average_loss = 0
# Note that this is expensive (~20% slowdown if computed every 500 steps)
if step % 10000 == 0:
sim = similarity.eval(session=sess)
#print(valid_size)
for i in range(valid_size):
valid_word = words[valid_examples[i]]
#print("valid_word",valid_word)#16
top_k = 8 # number of nearest neighbors
nearest = (-sim[i, :]).argsort()[1:top_k + 1] #argsort函数返回的是数组值从小到大的索引值
#print("nearest",nearest,top_k)
log_str = 'Nearest to %s:' % valid_word
for k in range(top_k):
close_word = words[nearest[k]]
log_str = '%s,%s' % (log_str, close_word)
print(log_str)
final_embeddings = normalized_embeddings.eval()
def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
plt.figure(figsize=(18, 18)) # in inches
for i, label in enumerate(labels):
x, y = low_dim_embs[i, :]
plt.scatter(x, y)
plt.annotate(label,xy=(x, y),xytext=(5, 2), textcoords='offset points',
ha='right',va='bottom')
plt.savefig(filename)
try:
# pylint: disable=g-import-not-at-top
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
plot_only = 80#输出100个词
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [words[i] for i in range(plot_only)]
#print(labels)
plot_with_labels(low_dim_embs, labels)
except ImportError:
print('Please install sklearn, matplotlib, and scipy to show embeddings.')