AI实战:搭建带注意力机制的 seq2seq 模型来做数值预测
seq2seq 框架图
环境依赖
- Linux
- python3.6
- tensorflow.keras
源码搭建模型及说明
- 依赖库
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import backend as K
from tensorflow.keras import activations
from tensorflow.keras.layers import Layer, Input, Embedding, LSTM, Dense, Attention, GRU, Dropout
from tensorflow.keras.models import Model
import numpy as np
- 编码
使用GRU作为特征提取器(针对数据量相对较少的情况,若是数据量很大,建议使用LSTM)
class Encoder(keras.Model):
def __init__(self, hidden_units):
super(Encoder, self).__init__()
# Encode LSTM Layer
self.encoder_lstm = GRU(hidden_units, return_sequences=True, return_state=True, name="encode_lstm")
self.dropout = Dropout(rate=0.5)
def call(self, inputs):
encoder_outputs, state_h = self.encoder_lstm(inputs)
return encoder_outputs, state_h
- 解码
解码结构中添加了注意力机制
class Decoder(keras.Model):
def __init__(self, hidden_units):
super(Decoder, self).__init__()
# Decode LSTM Layer
self.decoder_lstm = GRU(hidden_units, return_sequences=True, return_state=True, name="decode_lstm")
# Attention Layer
self.attention = Attention()
self.dropout = Dropout(rate=0.5)
def call(self, enc_outputs, dec_inputs, states_inputs):
dec_outputs, dec_state_h = self.decoder_lstm(dec_inputs, initial_state=states_inputs)
attention_output = self.attention([dec_outputs, enc_outputs])
return attention_output, dec_state_h
- 搭建带注意力机制的 seq2seq 模型
loss函数可以是tf自带的loss函数,也可以是自定义,如下面任选一个:
loss_fn = tf.keras.losses.MeanAbsoluteError()
loss_fn = mae
def mae(y_true, y_pred):
return K.mean(K.abs(y_pred - y_true))
整体代码如下:
def seq2seq_attention(encode_shape, decode_shape, hidden_units, output_dim):
"""
带注意力机制的seq2seq 模型
"""
# Input Layer
encoder_inputs = Input(shape=encode_shape, name="encode_input")
decoder_inputs = Input(shape=decode_shape, name="decode_input")
# Encoder Layer
encoder = Encoder(hidden_units)
enc_outputs, enc_state_h = encoder(encoder_inputs)
dec_states_inputs = enc_state_h
# Decoder Layer
decoder = Decoder(hidden_units)
attention_output, dec_state_h = decoder(enc_outputs, decoder_inputs, dec_states_inputs)
# Dense Layer
dense_outputs = Dense(output_dim, activation='sigmoid', name="dense")(Dropout(rate=0.5)(attention_output))
# seq2seq model
model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=dense_outputs)
model.summary()
opt = keras.optimizers.Adam(lr=0.0005)
#loss_fn = tf.keras.losses.MeanAbsolutePercentageError()
#loss_fn = tf.keras.losses.MeanAbsoluteError()
loss_fn = mae
model.compile(loss=loss_fn, optimizer=opt)
return model
- 测试搭建的模型
if __name__ == '__main__':
seq2seq_attention( (72, 21), (24, 20), 50, 1 )
-
最后
根据业务需求,生成对应的数据X1,X2,Y,分别对应encoder的输入、decoder的输入、输出 -
训练函数
def train(train_data_path, test_data_path):
batch_size = 512
epochs = 1000
X1, X2, Y = create_dataset(train_data_path)
train_data, eval_data, y_train, y_eval = split_data(X1, X2, Y, test_size=0.2, shuffle=shuffle)
#搭建模型
model = seq2seq_attention(encode_shape, decode_shape, hidden_units, output_dim)
#训练模型
callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=model_path,
save_weights_only=True,
monitor='val_loss',
mode='min',
save_best_only=True)
model.fit(x = train_data,
y = y_train,
batch_size = batch_size,
epochs = epochs,
callbacks=[callback, checkpoint_callback],
verbose = 2,
shuffle = True,
validation_data = (eval_data, y_eval))
#模型测试
X1, X2, Y = create_dataset(test_data_path)
test_data = [X1, X2]
y_test = Y
scores = model.evaluate(test_data, y_test, verbose=0)
print(model.metrics_names, scores)