参数
d_model = 512 # 输入到网路的词向量维度\是embedding的维度\是PE的最后一维度
d_k = 64 # w_q\w_k\w_v的维度
Max_len = 40 # 最大句长
head_en = 8 # 多头注意力head的个数
# 前馈网络的参数,其中输入输出维度跟d_model一样就不设置了,直接用d_model
Hidden_size = 2048 # 中间层的全连接神经元个数dff
en_N = 6
de_N = 6
vocab_input_len = 7926
vocab_target_len = 12556
##########################################################################
batch_size = 128
data_path = "./data/e2c.xlsx"
PAD = 0
SOS = 1
EOS = 2
学习率是1e-5
训练部分的代码
import random
import torch
import torch.nn as nn
import torch.optim as optim
from tqdm import tqdm
from Transformer import Transformer
from config import *
from get_parm import *
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# trainning pram
optim_l_r = 1e-3
epochs = 80
def Show_sentence(sentence_list):
for word in sentence_list:
if word == "<EOS>":
print(word, end="")
break
print(word, end="")
print()
def get_loss(decoder_outputs, target):
decoder_outputs = torch.log(decoder_outputs)
# torch.nn.CrossEntropyLoss==torch.nn.NLLLoss+torch.nn.LogSoftmax,因此这里需要取对数
target = target.view(-1) # [batch_size*max_len]
decoder_outputs = decoder_outputs.view(batch_size * max_len_ch, -1)
return criterion(decoder_outputs, target)
transformer = Transformer()
if os.path.exists("./models/transformer_e2c.pkl"):
transformer.load_state_dict(torch.load("./models/transformer_e2c.pkl"))
optimizer = optim.Adam(transformer.parameters(), optim_l_r)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30, 60], gamma=0.01)
criterion = nn.NLLLoss(ignore_index=vocab_en.word2idx("<PAD>"), reduction="mean")
transformer.train()
transformer.to(device)
Loss_list = []
loss_min = float('inf')
for epoch in range(epochs):
progress_bar = tqdm(total=len(trainDataLoader), desc='Train Epoch {}'.format(epoch), unit='batch')
loss_all = 0
n = 0
for idx, (en_text, ch_text) in enumerate(trainDataLoader):
n += 1
optimizer.zero_grad()
en_text = en_text.to(device)
ch_text = ch_text.to(device)
output = transformer(en_text, ch_text)
loss_t = get_loss(output, ch_text)
loss_all += loss_t.item()
loss_t.backward()
optimizer.step()
if idx == 0:
predict = torch.argmax(output.data, dim=-1)
print("预测:", end="")
number = random.randint(0, int(batch_size // 2))
Show_sentence(mapping(predict[number], vocab_ch, reverse=True))
print("真实:", end="")
Show_sentence(mapping(ch_text[number], vocab_ch, reverse=True))
progress_bar.set_postfix({'loss': loss_all / (idx + 1), "lr": optimizer.param_groups[0]['lr']})
progress_bar.update()
Loss_list.append(loss_all / n)
with open("loss/loss_data.txt", "w", encoding="utf8") as f:
f.write(str(Loss_list))
if (loss_all / n) < loss_min:
torch.save(transformer.state_dict(), "./models/transformer_e2c.pkl")
loss_min = loss_all / n
progress_bar.close()
scheduler.step()
注意的是:
损失函数使用NLLloss
推理过程
import ast
import torch
from Transformer import Transformer
from config import *
from vocab import tokenizer, Vocab, mapping
def load_vocab(path):
with open(path, "r", encoding="utf8") as f:
rel = ast.literal_eval(f.readline()) # 字符串列表转回列表
return rel
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if __name__ == "__main__":
# 加载词汇表同时构造词汇表类
vocab_en = Vocab(load_vocab("./vocab/vocab_en.txt"), init=False)
vocab_ch = Vocab(load_vocab("./vocab/vocab_ch.txt"), init=False)
# 载入模型
transformer = Transformer()
transformer.load_state_dict(torch.load("./models/transformer_e2c.pkl", map_location=torch.device(device)))
transformer.eval()
# 构造编码器的输入
en_input = ["Tom has no siblings."]
en_token_sentence_list = tokenizer(en_input)
print("编码器的输入:", en_token_sentence_list)
vector_en_list = []
for sentence in en_token_sentence_list:
vector_en_list.append(mapping(sentence, vocab_en))
print("编码器输入word2idx:", vector_en_list)
vector_en_list = torch.tensor(vector_en_list, dtype=torch.int64)
# 构造解码器的输入端GO
ch_sentence = ["Tom"]
ch_token_sentence_list = tokenizer(ch_sentence, inference=True)
print("解码器的输入:", ch_token_sentence_list)
vector_ch_list = []
for sentence in ch_token_sentence_list:
vector_ch_list.append(mapping(sentence, vocab_ch))
print("解码器输入word2idx:", vector_ch_list)
vector_ch_list = torch.tensor(vector_ch_list, dtype=torch.int64)
for _ in range(Max_len - vector_ch_list.shape[1] + 1):
output_de = transformer(vector_en_list, vector_ch_list)
if torch.argmax(output_de.data[:, -1, :], dim=-1).item() == EOS:
vector_ch_list = torch.cat((vector_ch_list, torch.argmax(output_de.data[:, -1, :], dim=-1).unsqueeze(0)),
dim=-1)
break
vector_ch_list = torch.cat((vector_ch_list, torch.argmax(output_de.data[:, -1, :], dim=-1).unsqueeze(0)),
dim=-1)
rel = mapping(vector_ch_list[0], vocab_ch, reverse=True)
print("经过transformer的输出:", rel)
print("#" * 100)
print("输入:", en_input[0])
print("输出:", end="")
for word in rel:
if word == "<SOS>": continue
print(word, end="")
print()
不知道是不是哪里有问题,推理的结果很差,但是训练的效果看着好像还可以:
可能是我推理过程搭错了,或者是训练的还不够
推理过程我是初始化一个SOS作为开始符号,然后送进网络,得到的结果跟SOS拼接,然后在送进网络,重复一直到输出EOS或者达到最大长度
其中的推理结果如下:
还在寻找哪里有问题
总结
transformer进行训练时,在小数据集上效果很难练出来,同时需要训练的epoch更多,参数方面也很难调整;而使用时序rnn模块来训练时,在数据集较小上的结果很不错,需要的代价也少很多。很大原因是因为transformer是有教师机制来引导训练的。