imdb数据集_keras如何导入本地下载的imdb数据集？

最新推荐文章于 2021-01-09 22:22:44 发布

weixin_39774044

最新推荐文章于 2021-01-09 22:22:44 发布

阅读量944

点赞数 2

文章标签： imdb数据集

第一步下载数据集到本地

下载链接：https://pan.baidu.com/s/15belo01keGvFri43K8TxFQ

提取码：9h3u

存储位置：C:/用户/用户名/.keras/datasets

（用户名不同人不一样，可能电脑不一样存储位置也略有差异）

第二步导入数据集

import keras
import numpy as np
# load data
from keras.datasets import imdb
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)

----------

查看数据集是否导入正确

print(train_labels[0]) #1
print(max([max(sequence) for sequence in train_data])) #9999

----------

遇到的一些小问题以及解决办法：

若出现了几个问题，最后差不多是这样：raise ValueError("Object arrays cannot be loaded when " ValueError: Object arrays cannot be loaded ……

这说明numpy版本太高了，我一开始的版本是1.16.4，之后转换成了1.16.2

版本转换：

cmd输入xxxxxxxxxxxxxxxx numpy==1.16.2

xxxxxxxxxx为https://mirrors.tuna.tsinghua.edu.cn/help/pypi/中代码，可加快下载速度，直接复制，只需要将some-package改成numpy==1.16.2即可

第三步电影评论二分类完整代码示例

import keras
import numpy as np
# load data
from keras.datasets import imdb
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)
print(train_labels[0]) #1
print(max([max(sequence) for sequence in train_data])) #9999
# 将索引解码为单词，需要下载imdb_word_index.json至C:/用户/用户名/.keras/datasets
# 链接：https://pan.baidu.com/s/1kkmpXrr1tkFtg7D3LX_lcw 提取码：wzjw
word_index = imdb.get_word_index() #将单词映射为整数索引的字典
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()]) #键值颠倒，将整数索引映射为单词
decoded_review = ' '.join([reverse_word_index.get(i - 3, '?') for i in train_data[0]])
#print(decoded_review)

# 对列表进行one-shot编码，eg.将[3,5]转换成[0.0,0.0,1.0,0.0,1.0,0.0,0.0,0.0,...]
def vectorize_sequences(sequences, dimension=10000):
results = np.zeros((len(sequences), dimension))
for i, sequence in enumerate(sequences):
results[i,sequence] = 1. #将 results[i] 的指定索引设为 1
return results
# handle input data
x_train = vectorize_sequences(train_data)
x_test = vectorize_sequences(test_data)
#print(x_train[0]) #[0. 1. 1. ... 0. 0. 0.]
# handle output data
y_train = np.asarray(train_labels).astype('float32')
y_test = np.asarray(test_labels).astype('float32')
# 验证集预留
x_val = x_train[:10000]
partial_x_train = x_train[10000:]
y_val = y_train[:10000]
partial_y_train = y_train[10000:]

# build model
from keras import models
from keras import layers
model = models.Sequential()
model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))
model.add(layers.Dense(16, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
# train model
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])

history = model.fit(partial_x_train,
partial_y_train,
epochs=20,
batch_size=512,
validation_data=(x_val, y_val))
history_dict = history.history
#print(history_dict.keys()) #dict_keys(['val_loss', 'val_acc', 'loss', 'acc'])
# 绘制训练损失、验证损失、训练精度、验证精度
import matplotlib.pyplot as plt
# plot loss
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc)+1)
plt.plot(epochs, loss, 'bo', label='Training loss') #blue o
plt.plot(epochs, val_loss, 'b', label='Validation loss') #blue solid line
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()
# plot accuracy
plt.clf() #清空图像
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()