课程来源:Text classification with movie reviews
Text classification with movie reviews
二分类(1:好评,0:差评)构建电影评论的分类模型
from __future__ import absolute_import, division, print_function
import tensorflow as tf
from tensorflow import keras
from data.tools import load_Localdata
import numpy as np
print(tf.__version__)
1.13.1
1 下载和查看数据集
# 下载npz文件中的训练数据和测试数据
fname = 'imdb.npz'
path = '/home/brian/Documents/tensorflow-gpu/tensorflow-learning/data/imdb/'
(train_data, train_labels), (test_data, test_labels) \
= load_Localdata(fname=fname, path=path, num_words=10000)
# 查看数据集的结构
print(type(train_data))
print("train_data.shape: ", train_data.shape)
print("train_labels.shape: ", train_labels.shape)
print("test_data.shape: ", test_data.shape)
print("test_labels.shape: ", test_labels.shape)
<class 'numpy.ndarray'>
train_data.shape: (25000,)
train_labels.shape: (25000,)
test_data.shape: (25000,)
test_labels.shape: (25000,)
训练数据集和测试数据集都有25000个
print("Training entries: {}, labels: {}".format(len(train_data), len(train_labels)))
Training entries: 25000, labels: 25000
print(type(train_data[0]))
print(train_data[0])
<class 'list'>
[1, 14, 22, 16, 43, 530, 973, 1622, 1385, 65, 458, 4468, 66, 3941, 4, 173, 36, 256, 5, 25, 100, 43, 838, 112, 50, 670, 2, 9, 35, 480, 284, 5, 150, 4, 172, 112, 167, 2, 336, 385, 39, 4, 172, 4536, 1111, 17, 546, 38, 13, 447, 4, 192, 50, 16, 6, 147, 2025, 19, 14, 22, 4, 1920, 4613, 469, 4, 22, 71, 87, 12, 16, 43, 530, 38, 76, 15, 13, 1247, 4, 22, 17, 515, 17, 12, 16, 626, 18, 2, 5, 62, 386, 12, 8, 316, 8, 106, 5, 4, 2223, 5244, 16, 480, 66, 3785, 33, 4, 130, 12, 16, 38, 619, 5, 25, 124, 51, 36, 135, 48, 25, 1415, 33, 6, 22, 12, 215, 28, 77, 52, 5, 14, 407, 16, 82, 2, 8, 4, 107, 117, 5952, 15, 256, 4, 2, 7, 3766, 5, 723, 36, 71, 43, 530, 476, 26, 400, 317, 46, 7, 4, 2, 1029, 13, 104, 88, 4, 381, 15, 297, 98, 32, 2071, 56, 26, 141, 6, 194, 7486, 18, 4, 226, 22, 21, 134, 476, 26, 480, 5, 144, 30, 5535, 18, 51, 36, 28, 224, 92, 25, 104, 4, 226, 65, 16, 38, 1334, 88, 12, 16, 283, 5, 16, 4472, 113, 103, 32, 15, 16, 5345, 19, 178, 32]
每个数据由列表组成,列表中存放评价的每个字符的标记
print(train_labels)
[1 0 0 ... 0 1 0]
二分类标签(0,1),其中1代表好评,0代表差评
len(train_data[0]), len(train_data[1])
(218, 189)
每个数据列表中存放的字符不一致,即评论都不是一样长,符合常识
将以上的整数字符转换为实义字符(真实字符)
from tensorflow.python.keras.utils.data_utils import get_file
import json
def get_word_index(path=None):
with open(path) as f:
return json.load(f)
# json中存放着字典,键为实义字符,值为字符的标记
path = '/home/brian/Documents/tensorflow-gpu/tensorflow-learning/data/imdb/imdb_word_index.json'
word_index = get_word_index(path=path)
# 所有编码加3,空出0-3的字符标记的位置
word_index = {k:(v+3) for k,v in word_index.items()}
# 将空出的标记用于以下字符
word_index["<PAD>"] = 0
word_index["<START>"] = 1 #开始字符
word_index["<UNK>"] = 2 #未知字符
word_index["<UNUSED>"] = 3#未使用字符
# 将字典的键值换位
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
# 构建字符标记到字符的转换函数
def decode_review(text):
return ' '.join([reverse_word_index.get(i, '?') for i in text])
# 将所有评论数据都填充到256大小,不足256的评论后填充word_index["<PAD>"],(本示例中为0)
train_data = keras.preprocessing.sequence.pad_sequences(train_data,
maxlen=256,
value=word_index["<PAD>"],
padding='post')
test_data = keras.preprocessing.sequence.pad_sequences(test_data,
maxlen=256,
value=word_index["<PAD>"],
padding='post')
print(type(train_data))
print(type(train_data[0]))
len(train_data[0]), len(test_data[0])
<class 'numpy.ndarray'>
<class 'numpy.ndarray'>
(256, 256)
2 构建模型
# 不同字符的总数
vocab_size = 10000
model = keras.Sequential()
model.add(keras.layers.Embedding(vocab_size, 16))
model.add(keras.layers.GlobalAveragePooling1D())
model.add(keras.layers.Dense(16, activation=tf.nn.relu))
model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))
model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, None, 16) 160000
_________________________________________________________________
global_average_pooling1d (Gl (None, 16) 0
_________________________________________________________________
dense (Dense) (None, 16) 272
_________________________________________________________________
dense_1 (Dense) (None, 1) 17
=================================================================
Total params: 160,289
Trainable params: 160,289
Non-trainable params: 0
3 训练模型
# 设定函数的优化算法、损失函数、指标
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
# 划分数据集为训练数据和验证数据
X_dev = train_data[:1000]
X_train = train_data[1000:]
Y_dev = train_labels[:1000]
Y_train = train_labels[1000:]
# 训练模型拟合训练数据
history = model.fit(X_train,Y_train,batch_size=512,epochs=40,
validation_data=(X_dev, Y_dev), verbose=1)
Epoch 40/40
24000/24000 [==============================] - 0s 5us/sample - loss: 0.0855 - acc: 0.9766 - val_loss: 0.3204 - val_acc: 0.8870
4 评估模型
loss, metrics = model.evaluate(test_data, test_labels)
print("test loss: ", loss)
print("metrics(acc): ", metrics)
25000/25000 [==============================] - 0s 20us/sample - loss: 0.3762 - acc: 0.8689
test loss: 0.3761545661497116
metrics(acc): 0.86888
5 可视化优化过程
history_dict = history.history
print(len(history_dict['loss']))
print(history_dict.keys())
40
dict_keys(['loss', 'acc', 'val_loss', 'val_acc'])
history.history是字典,存放着40次迭代的过程的数值
import matplotlib.pyplot as plt
acc = history_dict['acc']
val_acc = history_dict['val_acc']
loss = history_dict['loss']
val_loss = history_dict['val_loss']
epochs = range(1, len(acc)+1)
# bo 是 blue dot
plt.plot(epochs, loss, 'bo', label='Training loss')
# b 是 blue line
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend() #放置一个图例
plt.show()
plt.clf() # clear figure
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.xlabel('epochs')
plt.ylabel('accuracy')
plt.legend()
plt.show()
总结:从以上面的曲线可以看出模型过拟合,需要引入过拟合的措施