自然语言处理实验报告-实验14

使用Transformer实现序列到序列的机器翻译

机器翻译是自然语言处理领域的重要应用之一，通过计算机系统将一种语言的文本转换为另一种语言。本文将介绍如何使用Transformer模型实现中日文之间的机器翻译，通过Python和PyTorch实现该过程。

1. 数据准备

首先，我们需要准备用于训练的数据。我们使用了包含中日文对照的语料库，并使用SentencePiece进行分词处理。其中，中文和日文分别使用了不同的SentencePiece模型进行分词。

import pandas as pd

import sentencepiece as spm

# 读取数据

df = pd.read_csv('zh-ja.bicleaner05.txt', sep='\\t', engine='python', header=None)

trainen = df[2].values.tolist()

trainja = df[3].values.tolist()

# 加载SentencePiece模型

en_tokenizer = spm.SentencePieceProcessor(model_file='spm.en.nopretok.model')

ja_tokenizer = spm.SentencePieceProcessor(model_file='spm.ja.nopretok.model')

# 构建词汇表

ja_vocab = build_vocab(trainja, ja_tokenizer)

en_vocab = build_vocab(trainen, en_tokenizer)

2. 构建Transformer模型

接下来，我们构建Transformer模型，该模型分为编码器和解码器两部分，并实现了位置编码、掩码操作等关键组件。

import torch

import torch.nn as nn

from torch.nn.utils.rnn import pad_sequence

from torch.utils.data import DataLoader

from torchtext.vocab import Vocab

from torch.nn import TransformerEncoder, TransformerDecoder, TransformerEncoderLayer, TransformerDecoderLayer

import math

class Seq2SeqTransformer(nn.Module):

    def __init__(self, num_encoder_layers: int, num_decoder_layers: int,

                 emb_size: int, src_vocab_size: int, tgt_vocab_size: int,

                 dim_feedforward:int = 512, dropout:float = 0.1):

        super(Seq2SeqTransformer, self).__init__()

        

        # TransformerEncoder部分

        encoder_layer = TransformerEncoderLayer(d_model=emb_size, nhead=NHEAD,

                                                dim_feedforward=dim_feedforward)

        self.transformer_encoder = TransformerEncoder(encoder_layer, num_layers=num_encoder_layers)

        

        # TransformerDecoder部分

        decoder_layer = TransformerDecoderLayer(d_model=emb_size, nhead=NHEAD,

                                                dim_feedforward=dim_feedforward)

        self.transformer_decoder = TransformerDecoder(decoder_layer, num_layers=num_decoder_layers)

        # 输出层

        self.generator = nn.Linear(emb_size, tgt_vocab_size)

        self.src_tok_emb = TokenEmbedding(src_vocab_size, emb_size)

        self.tgt_tok_emb = TokenEmbedding(tgt_vocab_size, emb_size)

        

        # 位置编码层

        self.positional_encoding = PositionalEncoding(emb_size, dropout=dropout)

    def forward(self, src: Tensor, tgt: Tensor, src_mask: Tensor,

                tgt_mask: Tensor, src_padding_mask: Tensor,

                tgt_padding_mask: Tensor, memory_key_padding_mask: Tensor):

                

        # 编码器部分

        src_emb = self.positional_encoding(self.src_tok_emb(src))

        tgt_emb = self.positional_encoding(self.tgt_tok_emb(tgt))

        

        # 解码器部分

        memory = self.transformer_encoder(src_emb, src_mask, src_padding_mask)

        outs = self.transformer_decoder(tgt_emb, memory, tgt_mask, None,

                                        tgt_padding_mask, memory_key_padding_mask)

        

        # 生成器部分

        return self.generator(outs)

    def encode(self, src: Tensor, src_mask: Tensor):

        return self.transformer_encoder(self.positional_encoding(

                            self.src_tok_emb(src)), src_mask)

    def decode(self, tgt: Tensor, memory: Tensor, tgt_mask: Tensor):

        return self.transformer_decoder(self.positional_encoding(

                          self.tgt_tok_emb(tgt)), memory,

                          tgt_mask)

class PositionalEncoding(nn.Module):

    def __init__(self, emb_size: int, dropout, maxlen: int = 5000):

        super(PositionalEncoding, self).__init__()

        

        # 计算位置编码矩阵

        den = torch.exp(- torch.arange(0, emb_size, 2) * math.log(10000) / emb_size)

        pos = torch.arange(0, maxlen).reshape(maxlen, 1)

        pos_embedding = torch.zeros((maxlen, emb_size))

        pos_embedding[:, 0::2] = torch.sin(pos * den)

        pos_embedding[:, 1::2] = torch.cos(pos * den)

        pos_embedding = pos_embedding.unsqueeze(-2)

        # 使用 dropout

        self.dropout = nn.Dropout(dropout)

        

        # 注册位置编码矩阵为模型的 buffer

        self.register_buffer('pos_embedding', pos_embedding)

    def forward(self, token_embedding: Tensor):

        # 返回经过 dropout 后的词嵌入张量加上位置编码

        return self.dropout(token_embedding + self.pos_embedding[:token_embedding.size(0), :])

class TokenEmbedding(nn.Module):

    def __init__(self, vocab_size: int, emb_size):

        super(TokenEmbedding, self).__init__()

        

        # 创建词嵌入层

        self.embedding = nn.Embedding(vocab_size, emb_size)

        self.emb_size = emb_size

    def forward(self, tokens: Tensor):

        return self.embedding(tokens.long()) * math.sqrt(self.emb_size)

3. 训练模型

我们使用构建好的Transformer模型进行训练，定义损失函数和优化器，并迭代多个epoch进行模型优化。

NUM_ENCODER_LAYERS = 3

NUM_DECODER_LAYERS = 3

EMB_SIZE = 512

NHEAD = 8

FFN_HID_DIM = 512

NUM_EPOCHS = 16

# 初始化Transformer模型

transformer = Seq2SeqTransformer(NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS,

                                 EMB_SIZE, SRC_VOCAB_SIZE, TGT_VOCAB_SIZE,

                                 FFN_HID_DIM)

# 使用Xavier初始化方法对Transformer模型的参数进行初始化

for p in transformer.parameters():

    if p.dim() > 1:

        nn.init.xavier_uniform_(p)

transformer = transformer.to(device)

# 定义交叉熵损失函数

loss_fn = torch.nn.CrossEntropyLoss(ignore_index=PAD_IDX)

# 定义优化器

optimizer = torch.optim.Adam(

    transformer.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9

)

# 训练模型

for epoch in tqdm.tqdm(range(1, NUM_EPOCHS+1)):

    start_time = time.time()

    train_loss = train_epoch(transformer, train_iter, optimizer)

    end_time = time.time()

    print(f"Epoch: {epoch}, Train loss: {train_loss:.3f}, Epoch time = {(end_time - start_time):.3f}s")

4. 模型推断和评估

训练完成后，我们可以使用训练好的模型进行翻译，并进行评估来验证模型的效果。

def greedy_decode(model, src, src_mask, max_len, start_symbol):

    # 实现贪婪解码算法

    ...

def translate(model, src, src_vocab, tgt_vocab, src_tokenizer):

    # 实现翻译函数

    ...

# 示例翻译

translation = translate(transformer, "HSコード 8515 はんだ付け用、ろう付け用又は溶接用の機器(電気式(電気加熱ガス式を含む。)", ja_vocab, en_vocab, ja_tokenizer)

print(translation)

5. 结论

通过本文的介绍，我们了解了如何使用Transformer模型实现中日文之间的机器翻译。Transformer模型结合了编码器和解码器的设计，通过自注意力机制和位置编码有效地捕捉序列数据中的长距离依赖关系，提升了翻译质量和效率。