C5:Text,Sequences and Visualization

Scan

How to do it?

elems = np.array(["T","e","n","s","o","r"," ","F","l","o","w"])
scan_sum = tf.scan(lambda a, x:a+x,elems)
sess = tf.InteractiveSession()
print(sess.run(scan_sum))

RNN and Tensorboard

mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
element_size = 28
time_steps = 28
num_classes = 10
batch_size = 128
hidden_layer_size = 128
LOG_DIR = "logs/RNN_with_summaries"
_inputs = tf.placeholder(tf.float32,shape=[None, time_steps, element_size],name='inputs')
y = tf.placeholder(tf.float32, shape=[None, num_classes], name='labels')
def variable_summaries(var):
    with tf.name_scope('summaries'):
        mean = tf.reduce_mean(var)
        tf.summary.scalar('mean', mean)
        with tf.name_scope('stddev'):
            stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
        tf.summary.scalar('stddev', stddev)
        tf.summary.scalar('max', tf.reduce_max(var))
        tf.summary.scalar('min', tf.reduce_min(var))
        tf.summary.histogram('histogram', var)
#weight and bias for input and hidden layer
with tf.name_scope('rnn_weights'):
    with tf.name_scope("W_x"):
        Wx = tf.Variable(tf.zeros([element_size, hidden_layer_size]))
        variable_summaries(Wx)
    with tf.name_scope("W_h"):
        Wh = tf.Variable(tf.zeros([hidden_layer_size, hidden_layer_size]))
        variable_summaries(Wh)
    with tf.name_scope("Bias"):
        b_rnn = tf.Variable(tf.zeros([hidden_layer_size]))
        variable_summaries(b_rnn)
def rnn_step(previous_hidden_state, x):
        current_hidden_state = tf.tanh(
            tf.matmul(previous_hidden_state, Wh) +
            tf.matmul(x, Wx) + b_rnn)
        return current_hidden_state
processed_input = tf.transpose(_inputs, perm=[1, 0, 2])
initial_hidden = tf.zeros([batch_size, hidden_layer_size])
all_hidden_states = tf.scan(rnn_step, processed_input,initializer=initial_hidden,name='states')
with tf.name_scope('linear_layer_weights') as scope:
    with tf.name_scope("W_linear"):
        Wl = tf.Variable(tf.truncated_normal([hidden_layer_size, num_classes],mean=0, stddev=.01))
        variable_summaries(Wl)
    with tf.name_scope("Bias_linear"):
        bl = tf.Variable(tf.truncated_normal([num_classes],mean=0, stddev=.01))
        variable_summaries(bl)
def get_linear_layer(hidden_state):
    return tf.matmul(hidden_state, Wl) + bl
#weight for output layer
with tf.name_scope('linear_layer_weights') as scope:
    all_outputs = tf.map_fn(get_linear_layer, all_hidden_states)
    output = all_outputs[-1]
    tf.summary.histogram('outputs', output)
with tf.name_scope('cross_entropy'):
    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y))
    tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
    train_step = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cross_entropy)
with tf.name_scope('accuracy'):
    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(output, 1))
    accuracy = (tf.reduce_mean(tf.cast(correct_prediction, tf.float32)))*100
    tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
test_data = mnist.test.images[:batch_size].reshape((-1, time_steps, element_size))
test_label = mnist.test.labels[:batch_size]
with tf.Session() as sess:
    train_writer = tf.summary.FileWriter(LOG_DIR + '/train',graph=tf.get_default_graph())
    test_writer = tf.summary.FileWriter(LOG_DIR + '/test',graph=tf.get_default_graph())
    sess.run(tf.global_variables_initializer())
    for i in range(10000):
            batch_x, batch_y = mnist.train.next_batch(batch_size)
            batch_x = batch_x.reshape((batch_size, time_steps, element_size))
            summary, _ = sess.run([merged, train_step],feed_dict={_inputs: batch_x, y: batch_y})
            train_writer.add_summary(summary, i)
            if i % 1000 == 0:
                acc, loss, = sess.run([accuracy, cross_entropy],feed_dict={_inputs: batch_x,y: batch_y})
                print("Iter " + str(i) + ", Minibatch Loss= " +"{:.6f}".format(loss) + ", Training Accuracy= " +"{:.5f}".format(acc))
            if i % 100 == 0:
                summary, acc = sess.run([merged, accuracy],feed_dict={_inputs: test_data,y: test_label})
                test_writer.add_summary(summary, i)
    test_acc = sess.run(accuracy, feed_dict={_inputs: test_data,y: test_label})
    print("Test Accuracy:", test_acc)

Display:

BasicRNN

mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
element_size = 28
time_steps = 28
num_classes = 10
batch_size = 128
hidden_layer_size = 128
_inputs = tf.placeholder(tf.float32,shape=[None, time_steps, element_size],name='inputs')
y = tf.placeholder(tf.float32, shape=[None, num_classes], name='inputs')
# TensorFlow built-in functions
rnn_cell = tf.contrib.rnn.BasicRNNCell(hidden_layer_size)
outputs, _ = tf.nn.dynamic_rnn(rnn_cell, _inputs, dtype=tf.float32)
Wl = tf.Variable(tf.truncated_normal([hidden_layer_size, num_classes],mean=0, stddev=.01))
bl = tf.Variable(tf.truncated_normal([num_classes], mean=0, stddev=.01))
def get_linear_layer(vector):
    return tf.matmul(vector, Wl) + bl
last_rnn_output = outputs[:, -1, :]
final_output = get_linear_layer(last_rnn_output)
softmax = tf.nn.softmax_cross_entropy_with_logits(logits=final_output, labels=y)
cross_entropy = tf.reduce_mean(softmax)
train_step = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(final_output, 1))
accuracy = (tf.reduce_mean(tf.cast(correct_prediction, tf.float32))) * 100
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
test_data = mnist.test.images[:batch_size].reshape((-1, time_steps, element_size))
test_label = mnist.test.labels[:batch_size]
for i in range(3001):
    batch_x, batch_y = mnist.train.next_batch(batch_size)
    batch_x = batch_x.reshape((batch_size, time_steps, element_size))
    sess.run(train_step, feed_dict={_inputs: batch_x,y: batch_y})
    if i % 1000 == 0:
        acc = sess.run(accuracy, feed_dict={_inputs: batch_x,y: batch_y})
        loss = sess.run(cross_entropy, feed_dict={_inputs: batch_x,y: batch_y})
        print("Iter " + str(i) + ", Minibatch Loss={:.6f}".format(loss) + ", Training Accuracy= {:.5f}".format(acc))
print("Testing Accuracy:", sess.run(
    accuracy, feed_dict={_inputs: test_data, y: test_label}))

Display

LSTM

batch_size = 128
embedding_dimension = 64
num_classes = 2
hidden_layer_size = 32
times_steps = 6
element_size = 1
digit_to_word_map = {1: "One", 2: "Two", 3: "Three", 4: "Four", 5: "Five",6: "Six", 7: "Seven", 8: "Eight", 9: "Nine"}
digit_to_word_map[0] = "PAD"
even_sentences = []
odd_sentences = []
seqlens = []
for i in range(10000):
    rand_seq_len = np.random.choice(range(3, 7))
    seqlens.append(rand_seq_len)
    rand_odd_ints = np.random.choice(range(1, 10, 2),rand_seq_len)
    rand_even_ints = np.random.choice(range(2, 10, 2),rand_seq_len)
    if rand_seq_len < 6:
        rand_odd_ints = np.append(rand_odd_ints,[0]*(6-rand_seq_len))
        rand_even_ints = np.append(rand_even_ints,[0]*(6-rand_seq_len))
    even_sentences.append(" ".join([digit_to_word_map[r] for r in rand_odd_ints]))
    odd_sentences.append(" ".join([digit_to_word_map[r] for r in rand_even_ints]))
data = even_sentences+odd_sentences
seqlens *= 2
labels = [1] * 10000 + [0] * 10000
for i in range(len(labels)):
    label = labels[i]
    one_hot_encoding = [0]*2
    one_hot_encoding[label] = 1
    labels[i] = one_hot_encoding
word2index_map = {}
index = 0
for sent in data:
    for word in sent.lower().split():
        if word not in word2index_map:
            word2index_map[word] = index
            index += 1
index2word_map = {index: word for word, index in word2index_map.items()}
vocabulary_size = len(index2word_map)
data_indices = list(range(len(data)))
np.random.shuffle(data_indices)
data = np.array(data)[data_indices]
labels = np.array(labels)[data_indices]
seqlens = np.array(seqlens)[data_indices]
train_x = data[:10000]
train_y = labels[:10000]
train_seqlens = seqlens[:10000]
test_x = data[10000:]
test_y = labels[10000:]
test_seqlens = seqlens[10000:]
def get_sentence_batch(batch_size, data_x,
                       data_y, data_seqlens):
    instance_indices = list(range(len(data_x)))
    np.random.shuffle(instance_indices)
    batch = instance_indices[:batch_size]
    x = [[word2index_map[word] for word in data_x[i].lower().split()]for i in batch]
    y = [data_y[i] for i in batch]
    seqlens = [data_seqlens[i] for i in batch]
    return x, y, seqlens
_inputs = tf.placeholder(tf.int32, shape=[batch_size, times_steps])
_labels = tf.placeholder(tf.float32, shape=[batch_size, num_classes])
_seqlens = tf.placeholder(tf.int32, shape=[batch_size])
with tf.name_scope("embeddings"):
    embeddings = tf.Variable(tf.random_uniform([vocabulary_size,embedding_dimension],-1.0, 1.0), name='embedding')
    embed = tf.nn.embedding_lookup(embeddings, _inputs)
with tf.variable_scope("lstm"):
    lstm_cell = tf.contrib.rnn.BasicLSTMCell(hidden_layer_size,forget_bias=1.0)
    outputs, states = tf.nn.dynamic_rnn(lstm_cell, embed,sequence_length=_seqlens,dtype=tf.float32)
weights = {
    'linear_layer': tf.Variable(tf.truncated_normal([hidden_layer_size, num_classes],mean=0, stddev=.01))
}
biases = {
    'linear_layer': tf.Variable(tf.truncated_normal([num_classes], mean=0, stddev=.01))
}
final_output = tf.matmul(states[1],weights["linear_layer"]) + biases["linear_layer"]
softmax = tf.nn.softmax_cross_entropy_with_logits(logits=final_output,labels=_labels)
cross_entropy = tf.reduce_mean(softmax)
train_step = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(_labels, 1),tf.argmax(final_output, 1))
accuracy = (tf.reduce_mean(tf.cast(correct_prediction,tf.float32)))*100
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for step in range(1000):
        x_batch, y_batch, seqlen_batch = get_sentence_batch(batch_size,train_x, train_y,train_seqlens)
        sess.run(train_step, feed_dict={_inputs: x_batch, _labels: y_batch,_seqlens: seqlen_batch})
        if step % 100 == 0:
            acc = sess.run(accuracy, feed_dict={_inputs: x_batch,_labels: y_batch,_seqlens: seqlen_batch})
            print("Accuracy at %d: %.5f" % (step, acc))
    for test_batch in range(5):
        x_test, y_test, seqlen_test = get_sentence_batch(batch_size,test_x, test_y,test_seqlens)
        batch_pred, batch_acc = sess.run([tf.argmax(final_output, 1), accuracy],feed_dict={_inputs: x_test,_labels: y_test,_seqlens: seqlen_test})
        print("Test batch accuracy %d: %.5f" % (test_batch, batch_acc))
    output_example = sess.run([outputs], feed_dict={_inputs: x_test,_labels: y_test,_seqlens: seqlen_test})
    states_example = sess.run([states[1]], feed_dict={_inputs: x_test,_labels: y_test, _seqlens: seqlen_test})

Display