Embedding是神经网络用于NLP的关键,能够把词语转化成数字
解决问题:序列式问题
经典模型:LSTM
二 Embedding
NLP领域中,onehot编码应用不广泛,常用Dense embedding。
区别:
One-hot:Word -> index -> [0,0,1,...,0,0]
Dense emdeding:Word -> index -> [1.2, 4.2 ...] 初始化是随机的
处理变长输入:Padding -> [3,2,3,5,6] -> []3,2,3,5,6,0,0,0,0,0], 0表示Unkown,padding处理短句子,截断处理长句子。
输入长度不同,emdeding层则通过padding和intercept使得,每个输入长度都相同。处理完成后再进行合并。
缺点:
信息丢失,有很多padding(噪音),合并会使得词语没有主次,顺序之分。
无效计算太多。
三 为何需要循环神经网络?
解决序列式问题:
一对多:图片生成描述,一句话描述一张图片
多对一:文本分类(文本情感分析)
多对多:机器翻译,非实时的多对多问题,一句话全部输入完才会有输出
实时多对多:视频解说
四 循环神经网络
训练过程(BPTT):
预测过程:
将前一个RNN的输出,作为下一个的输入。
五 简单RNN文本生成
import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
import requests
def getWebPage(url):
try:
urlpage = requests.get(url)
except IOError:
print("IOError")
'''
urlpage.text中包含网页的源码内容
'''
WebPageDownload(urlpage.text)
def WebPageDownload(text):
'''
将下载的网页保存到file.txt文件中
'''
ff = open("shakespeare.txt",'w')
ff.writelines(text)
ff.close()
getWebPage(url='https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')
input_filepath = "shakespeare.txt"
text = open(input_filepath, 'r').read()
print(len(text))
print(text[0:100])
# 1. generate vocab 生成词表
# 2. build mapping char->id 建立映射
# 3. data -> id_data 数据转成id
# 4. abcd -> bcd<eos> 定义输入输出
vocab = sorted(set(text))
print(len(vocab))
print(vocab)
char2idx = {char:idx for idx, char in enumerate(vocab)}
print(char2idx)
idx2char = np.array(vocab)
print(idx2char)
# 文本的id列表
text_as_int = np.array([char2idx[c] for c in text])
print(text_as_int[0:10])
print(text[0:10])
def split_input_target(id_text):
"""
abcde -> abcd, bcde
"""
return id_text[0:-1], id_text[1:]
char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)
seq_length = 100
# 转化为sequence
# drop_remainder 最后一个batch不够被丢弃
seq_dataset = char_dataset.batch(seq_length + 1,
drop_remainder = True)
for ch_id in char_dataset.take(2):
print(ch_id, idx2char[ch_id.numpy()])
for seq_id in seq_dataset.take(2):
print(seq_id)
print(repr(''.join(idx2char[seq_id.numpy()])))
# 生成输入输出
seq_dataset = seq_dataset.map(split_input_target)
for item_input, item_output in seq_dataset.take(2):
print(item_input.numpy())
print(item_output.numpy())
batch_size = 64
buffer_size = 10000
seq_dataset = seq_dataset.shuffle(buffer_size).batch(
batch_size, drop_remainder=True)
# 预测下一个单词的概率,需要知道有多少个单词
vocab_size = len(vocab)
embedding_dim = 256
# RNN 宽度
rnn_units = 1024
def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
model = keras.models.Sequential([
keras.layers.Embedding(vocab_size, embedding_dim,
batch_input_shape = [batch_size, None]),
# return_sequences返回结果是个序列
keras.layers.SimpleRNN(units = rnn_units,
stateful = True,
recurrent_initializer = 'glorot_uniform',
return_sequences = True),
keras.layers.Dense(vocab_size),
])
return model
model = build_model(
vocab_size = vocab_size,
embedding_dim = embedding_dim,
rnn_units = rnn_units,
batch_size = batch_size)
model.summary()
for input_example_batch, target_example_batch in seq_dataset.take(1):
example_batch_predictions = model(input_example_batch)
print(example_batch_predictions.shape)
# 两种不同生成结果的策略
# random sampling. 上面代码产生了一个分布,根据分布来进行随机采样,可以得到多个序列
# greedy, random. 贪心选择最大概率的,只会产生一个序列
sample_indices = tf.random.categorical(
logits = example_batch_predictions[0], num_samples = 1)
print(sample_indices)
# (100, 65) -> (100, 1)
# tf.squeeze压缩成一个(100, )
sample_indices = tf.squeeze(sample_indices, axis = -1)
print(sample_indices)
print("Input: ", repr("".join(idx2char[input_example_batch[0]])))
print()
print("Output: ", repr("".join(idx2char[target_example_batch[0]])))
print()
print("Predictions: ", repr("".join(idx2char[sample_indices])))
# 自定义 loss,在model中的最后一层没有进行softmax,所以loss中进行softmax并进行计算
def loss(labels, logits):
return keras.losses.sparse_categorical_crossentropy(
labels, logits, from_logits=True)
model.compile(optimizer = 'adam', loss = loss)
example_loss = loss(target_example_batch, example_batch_predictions)
print(example_loss.shape)
print(example_loss.numpy().mean())
output_dir = "./text_generation_checkpoints"
if not os.path.exists(output_dir):
os.mkdir(output_dir)
checkpoint_prefix = os.path.join(output_dir, 'ckpt_{epoch}')
checkpoint_callback = keras.callbacks.ModelCheckpoint(
filepath = checkpoint_prefix,
save_weights_only = True)
epochs = 100
history = model.fit(seq_dataset, epochs = epochs,
callbacks = [checkpoint_callback])
model2 = build_model(vocab_size,
embedding_dim,
rnn_units,
batch_size = 1)
# 使用saveweights来保存最佳的参数,以后方便载入
model2.load_weights(tf.train.latest_checkpoint(output_dir))
model2.build(tf.TensorShape([1, None]))
# start ch sequence A,
# A -> model -> b
# A.append(b) -> B
# B(Ab) -> model -> c
# B.append(c) -> C
# C(Abc) -> model -> ...
model2.summary()
def generate_text(model, start_string, num_generate = 1000):
input_eval = [char2idx[ch] for ch in start_string]
input_eval = tf.expand_dims(input_eval, 0)
text_generated = []
model.reset_states()
for _ in range(num_generate):
# 1. model inference -> predictions
# 2. sample -> ch -> text_generated.
# 3. update input_eval
# predictions : [batch_size, input_eval_len, vocab_size]
predictions = model(input_eval)
# predictions : [input_eval_len, vocab_size]
predictions = tf.squeeze(predictions, 0)
# predicted_ids: [input_eval_len, 1]
# a b c -> b c d
predicted_id = tf.random.categorical(
predictions, num_samples = 1)[-1, 0].numpy()
text_generated.append(idx2char[predicted_id])
# s, x -> rnn -> s', y
input_eval = tf.expand_dims([predicted_id], 0)
return start_string + ''.join(text_generated)
new_text = generate_text(model2, "All: ")
print(new_text)