keras 之 fit_generator 和 validation_data

欢迎关注公众号：pythonhonor，一起学习python。

摘自：Python深度学习

下载链接: https://pan.baidu.com/s/1GRBL5WvxHj79i9Rs28AshQ  提取码:  dpnf

直接上示例代码：

数据集下载地址：jena_climate_2009_2016.csv（https://s3.amazonaws.com/keras-datasets/jena_climate_2009_2016.csv.zip）

 

import os
data_dir = 'data/'
fname = os.path.join(data_dir, 'jena_climate_2009_2016.csv')

f = open(fname)
data = f.read()
f.close()

lines = data.split('\n')
header = lines[0].split(',')
lines = lines[1:]

print(header)
print(len(lines))

 

import numpy as np

float_data = np.zeros((len(lines), len(header) - 1))

for i, line in enumerate(lines):
    values = [float(x) for x in line.split(',')[1:]]
    float_data[i, :] = values

 

from matplotlib import pyplot as plt

temp = float_data[:, 1]
#print(plt.plot(range(len(temp)), temp))
print(plt.plot(range(1440), temp[:1440]))

 

mean = float_data[:200000].mean(axis=0)
float_data -= mean
std = float_data[:200000].std(axis=0)
float_data /= std

 

def generator(data, lookback, delay, min_index, max_index,
        shuffle=False, batch_size=128, step=6):
    if max_index is None:
        max_index = len(data) - delay - 1
    i = min_index + lookback

    while 1:
        if shuffle:
            rows = np.random.randint(
                min_index + lookback, max_index, size=batch_size)
        else:
            if i + batch_size >= max_index:
                i = min_index + lookback
            rows = np.arange(i, min(i + batch_size, max_index))
            i += len(rows)

        samples = np.zeros((len(rows),
            lookback // step,
            data.shape[-1]))
        targets = np.zeros((len(rows),))

        for j, row in enumerate(rows):
            indices = range(rows[j] - lookback, rows[j], step)
            samples[j] = data[indices]
            targets[j] = data[rows[j] + delay][1]

        yield samples, targets

 

lookback = 1440
step = 6
delay = 144
batch_size = 128

train_gen = generator(float_data,
    lookback=lookback,
    delay=delay,
    min_index=0,
    max_index=200000,
    shuffle=True,
    step=step,
    batch_size=batch_size)

val_gen = generator(float_data,
    lookback=lookback,
    delay=delay,
    min_index=200001,
    max_index=300000,
    step=step,
    batch_size=batch_size)

test_gen = generator(float_data,
    lookback=lookback,
    delay=delay,
    min_index=300001,
    max_index=None,
    step=step,
    batch_size=batch_size)

 

val_steps = (300000 - 200001 - lookback) //batch_size 
test_steps = (len(float_data) - 300001 - lookback) //batch_size

 

def evaluate_naive_method():
    batch_maes = []
    for step in range(val_steps):
        samples, targets = next(val_gen)
        preds = samples[:, -1, 1]
        mae = np.mean(np.abs(preds - targets))
        batch_maes.append(mae)
        
    print(np.mean(batch_maes))

evaluate_naive_method()

# 0.2897359729905486

 

celsius_mae = 0.29 * std[1]

 

from keras.models import Sequential 
from keras import layers
from keras.optimizers import RMSprop
model = Sequential()
model.add(layers.Flatten(input_shape=(lookback // step, float_data.shape[-1]))) 
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1))


model.compile(optimizer=RMSprop(), loss='mae') 
history = model.fit_generator(train_gen,
    steps_per_epoch=500, 
    epochs=10, 
    validation_data=val_gen, 
    validation_steps=val_steps)

运行结果：可以看出在每一轮训练完毕，会整体计算一个 “验证数据集”的指标 （val_loss）

 

import matplotlib.pyplot as plt

loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.show()