文本处理------embedding、分词、文本截断与填充

# embedding层至少需要两个参数：标记的个数(这里是1000，即最大单词索引+1)和嵌入的维度(这里是64)

# 输入是二维整数张量，(samples,sequence_length)，将单词索引(表示特定单词)映射为对应的词向量
# (32,10)表示32个长度为10的序列组成的批量，较长序列应被截断，较短序列以0填充

# 返回三维浮点数张量(samples,sequence_length,embedding_dimensionalty)
# 然后可以用RNN层或一维卷积层处理这个三维张量

# embedding后跟全连接层，需展平model.add(layers.Flatten())
# embedding后跟RNN，不需要展平
# RNN后跟全连接，不需要展平
# 卷积/池化后跟全连接，需展平

from keras.layers import Embedding
embedding_layer = Embedding(1000, 64)

# 分词
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequence
import numpy as np

maxlen = 100
training_samples = 200
validation_samples = 10000
max_words = 10000

tokenizer = Tokenizer(num_words=max_words)
tokenizer,fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(text)

word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))

data = pad_sequences(sequences,maxlen=maxlen)

labels = np.asarray(labels)
indices = np.arange(data.shap[0])
np.random.shuffle(indices)

data = data[indices)
labels = labels[indices]

x_train = data[:training_samples]
y_train = labels[:training_samples]
x_val = data[training_samples:training_samples+validation_samples]
y_val = labels[training_samples:training_samples+validation_samples]

# 截断与填充文本
from keras.preprocessing.sequence import pad_sequence
x_train = pad_sequence(x_train, maxlen=20)