Transformer总结

1.Transform背景介绍

1.1Transform的优势

相比于之前占领市场的LSTM和GRU模型，Transformer有两个显著的优势：

（1）Transform能够使用分布式GPU进行并行训练，提升模型训练效率

（2） 在分析预测更长的文本时，捕捉间隔较长的语义关联效果更好

2.认识Transformer架构

2.1Transfromer模型的作用

1.基于seq2seq架构的Transform模型可以完成nlp领域研究的典型任务，如机器翻译，文本生成等，同时有可以构建预训练语言模型，用于不同任务的迁移学习

2.在接下来的架构分析中，我们将假设使用Transformer模型处理从一种语言文本到另一种语言文本的翻译工作，因此很多命名方式遵循nlp中的规则，比如:Embedding层称为文本嵌入层，Embedding层产生的张量称为词嵌入张量，它的最后一维将称作词向量等

2.2Transfromer总体架构图

2.3Transformer组成结构分析

​​​​​​3.输入部分实现

3.1输入部分组成

源文本嵌入层以及位置编码器

目标文本嵌入层以及位置编码器

3.2文本嵌入层的作用

无论是原文本嵌入还是目标文本嵌入，都是为了将文本中词汇的数字表示转变为向量表示，希望在这样的高维空间捕捉词汇间的关系

文本嵌入层代码

import torch
import torch.nn as nn
import math
from torch.autograd import Variable


class Embeddings(nn.Module):
    def __init__(self, d_model, vocab):
        super(Embeddings, self).__init__()
        # 参数d_model  每个词汇的特征尺寸  词嵌入维度
        # 参数vocab  词汇表大小
        self.d_model = d_model
        self.vocab = vocab
        self.lut = nn.Embedding(self.vocab, self.d_model)

    def forward(self, x):
        return self.lut(x) * math.sqrt(self.d_model)


# embedding = nn.Embedding(10, 3)
# input = torch.LongTensor([[1, 2, 4, 5], [4, 3, 2, 9]])
# print(embedding(input))
# embedding = nn.Embedding(10, 3, padding_idx=0)
# input = torch.LongTensor([0, 2, 0, 5])
# print(embedding(input))
# 调用
def dm_test_Embeddings():
    d_model = 512
    vocab = 1000
    my_embeddings = Embeddings(d_model, vocab)
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    embed = my_embeddings(x)
    print('embed.shape',embed.shape,'\nembed--->\n',embed)
#dm_test_Embeddings()

3.3位置编码器

位置编码器的作用：在Transformer的编码器结构中，并没有针对词汇位置信息的处理，因此需要再Embedding层后加入位置编码器，将词汇位置不同可能会产生的不同语义的信息加入到词嵌入张量中，以弥补位置信息的缺失

位置编码器代码

import torch
import torch.nn as nn
import math
from torch.autograd import Variable
from 文本嵌入层 import Embeddings


class PositionalEncoding(nn.Module):
    def __init__(self, d_model, dropout, max_len=5000):
        # 参数d_model 词嵌入维度 eg: 512个特征
        # 参数max_len 单词token个数 eg: 60个单词
        super(PositionalEncoding, self).__init__()

        # 定义dropout层
        self.dropout = nn.Dropout(p=dropout)

        # 思路：位置编码矩阵 + 特征矩阵 相当于给特征增加了位置信息
        # 定义位置编码矩阵PE eg pe[60, 512], 位置编码矩阵和特征矩阵形状是一样的
        pe = torch.zeros(max_len, d_model)

        # 定义位置列-矩阵position  数据形状[max_len,1] eg: [0,1,2,3,4...60]^T
        position = torch.arange(0, max_len).unsqueeze(1)
        # print('position--->', position.shape, position)

        # 定义变化矩阵div_term [1,256]
        # torch.arange(start=1, end=512, 2)结果并不包含end。在start和end之间做一个等差数组 [0, 2, 4, 6 ... 510]
        div_term = torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model))

        # 位置列-矩阵 @ 变化矩阵 做矩阵运算 [60*1]@ [1*256] ==> 60 *256
        # 矩阵相乘也就是行列对应位置相乘再相加，其含义，给每一个列属性（列特征）增加位置编码信息
        my_matmulres = position * div_term
        # print('my_matmulres--->', my_matmulres.shape, my_matmulres)

        # 给位置编码矩阵奇数列，赋值sin曲线特征
        pe[:, 0::2] = torch.sin(my_matmulres)
        # 给位置编码矩阵偶数列，赋值cos曲线特征
        pe[:, 1::2] = torch.cos(my_matmulres)

        # 形状变化 [60,512]-->[1,60,512]
        pe = pe.unsqueeze(0)

        # 把pe位置编码矩阵 注册成模型的持久缓冲区buffer; 模型保存再加载时，可以根模型参数一样，一同被加载
        # 什么是buffer: 对模型效果有帮助的，但是却不是模型结构中超参数或者参数，不参与模型训练
        self.register_buffer('pe', pe)

    def forward(self, x):
        # 注意：输入的x形状2*4*512  pe是1*60*512 形状 如何进行相加
        # 只需按照x的单词个数 给特征增加位置信息
        x = x + Variable(self.pe[:, :x.size()[1]], requires_grad=False)
        return self.dropout(x)


# 调用
def dm_test_PositionalEncoding():
    d_model = 512
    vocab = 1000
    # 实例化词嵌入层
    my_embeddings = Embeddings(d_model, vocab)
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    embed = my_embeddings(x)
    my_pe = PositionalEncoding(d_model=d_model, dropout=0.1, max_len=60)
    # 给词嵌入数据embed 添加位置特征
    pe_result = my_pe(embed)
    print('pe_result.shape--->', pe_result.shape)
    print('pe_result--->', pe_result)
dm_test_PositionalEncoding()

3.4绘制词汇向量中特征的分布曲线

代码

import matplotlib.pyplot as plt
import numpy as np
import torch
from torch.autograd import Variable

from 位置编码器 import PositionalEncoding


# 绘制PE位置特征sin-cos曲线
def dm_draw_PE_feature():
    my_pe = PositionalEncoding(d_model=20, dropout=0)
    print('my_positionalencoding.shape-->', my_pe.pe.shape)
    y = my_pe(Variable(torch.zeros(1, 100, 20)))
    print('y-->', y.shape)
    plt.figure(figsize=(20, 20))
    plt.plot(np.arange(100), y[0, :, 4:8].numpy())
    plt.legend(['dim %d' %p for p in [4, 5, 6, 7]])
    plt.show()
dm_draw_PE_feature()

4.编码器部分的实现

4.1编码器介绍

编码器部分：由N个编码器层堆叠而成，每个编码器层由两个子层连接结构组成。第一个子层连接结构包括一个多头自注意力子层，规范化层和一个残差连接层，第二个子层连接结构包括一个前馈全连接层，规范化层和一个残差连接层

4.2掩码张量的介绍

掩码张量的作用：掩码的作用就是让另外一个张量的一些数值被遮掩

4.3掩码张量函数

掩码张量函数：subsquent_mask

它的输入是size，代表掩码张量的大小

它的输出是一个下三角阵

4.4注意力机制

注意力计算规则：Attention(Q,K,V)=Softmax(Q⋅KT/√dk)⋅V

注意力计算规则的代码分析

import math
import torch
import torch.nn.functional as F
from torch.autograd import Variable
from 文本嵌入层 import Embeddings
from 位置编码器 import PositionalEncoding


def attention(query, key, value, mask=None, dropout=None):
    d_k = query.size()[-1]
    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)
    if mask is not None:
        scores = scores.masked_fill(mask == 0, -1e9)
    p_attn = F.softmax(scores, dim=-1)
    if dropout is not None:
        p_attn = dropout(p_attn)
    return torch.matmul(p_attn, value), p_attn


# 调用
def dm_test_attention():
    d_model = 512
    vocab = 1000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    my_embeddings = Embeddings(d_model, vocab)
    x = my_embeddings(x)
    drop_out = 0.1
    max_len = 60
    my_pe = PositionalEncoding(d_model, drop_out, max_len)
    pe_result = my_pe(x)
    query = key = value = pe_result
    atten1, patten1 = attention(query, key, value)
    print("编码阶段，对注意力权重不做掩码")
    print("atten1--->", atten1.shape, '\n', atten1)
    print("patten1-->", patten1.shape, '\n', patten1)
    print('*' * 50)
    print("编码阶段，对注意力权重做掩码")
    mask = Variable(torch.zeros(2, 4, 4))
    atten2,patten2=attention(query,key,value,mask=mask)
    print("atten2-->",atten2.shape,'\n',atten2)
    print("patten2-->",patten2.shape,'\n',patten2)
dm_test_attention()

4.5多头注意力机制

1.多头注意力机制的概念

对三个变换张量Q,K,V分别进行线性变换，这些变换不会改变原有张量的尺寸，因此每个变换矩阵都是方阵，得到输出结果后，多头的作用才开始显现，每个头开始从词义层面分割输出的张量，也就是每个头都想获得一组Q,K,V来进行注意力机制的计算，但句子中的每个词的表示只获得一部分，也就是只分割了最后一维的词嵌入向量，这就是所谓的多头，将每个头获得的输入送到注意力机制中，就形成了多头注意力机制

2.多头注意力机制的作用

这种结构设计能够让每个注意力机制去优化每个词汇的不同特征部分，从而均衡同一种注意力机制可能产生的偏差，让词义拥有来自更多元的表达，实验表明可以从而提升模型效果

3.多头注意力机制的代码实现

import copy
import torch
import torch.nn as nn
from 注意力机制 import attention
from torch.autograd import Variable
from 文本嵌入层 import Embeddings
from 位置编码器 import PositionalEncoding


def clones(moudle, N):
    return nn.ModuleList([copy.deepcopy(moudle) for _ in range(N)])


class MultiHeadAttention(nn.Module):
    def __init__(self, head, embedding_dim, dropout=0.1):
        super(MultiHeadAttention, self).__init__()
        assert embedding_dim % head == 0
        self.d_k = embedding_dim // head
        self.head = head
        self.linears = clones(nn.Linear(embedding_dim, embedding_dim), 4)
        self.attn = None
        self.dropout = nn.Dropout(p=dropout)

    def forward(self, query, key, value, mask=None):
        if mask is not None:
            mask = mask.unsqueeze(0)
        batch_size = query.size()[0]
        query, key, value = [model(x).view(batch_size, -1, self.head, self.d_k).transpose(1, 2)
                             for model, x in zip(self.linears, (query, key, value))]
        x, self.attn = attention(query, key, value, mask=mask, dropout=self.dropout)
        x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.head * self.d_k)
        return self.linears[-1](x)


# 测试多头注意力机制
def dm_test_MultiHeadedAttention():
    d_model = 512
    vocab = 1000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    my_embeddings = Embeddings(d_model, vocab)
    x = my_embeddings(x)
    dropout = 0.1
    max_len = 60
    my_pe = PositionalEncoding(d_model, dropout, max_len)
    pe_result = my_pe(x)
    head = 8
    query = key = value = pe_result
    #输入的掩码张量mask
    mask = Variable(torch.zeros(8, 4, 4))
    my_mha = MultiHeadAttention(head, d_model, dropout)
    x = my_mha(query, key, value, mask)
    print('多头注意力机制后的x',x.shape,'\n',x)
    print('多头注意力机制的注意力权重分布',my_mha.attn.shape)

dm_test_MultiHeadedAttention()

4.6前馈全连接层

1.概念：在Transformer中，前馈全连接层就是具有两层线性层的全连接网络

2.前馈全连接层的作用：考虑注意力机制可能对复杂过程的拟合程度不够，通过增加两层网络来增强模型的能力

前馈全连接层代码：

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from 多头注意力机制 import MultiHeadAttention
from 位置编码器 import PositionalEncoding
from  文本嵌入层 import  Embeddings


class PositionwiseFeedForward(nn.Module):
    def __init__(self, d_model, d_ff, dropput=0.1):
        super(PositionwiseFeedForward, self).__init__()
        self.w1 = nn.Linear(d_model, d_ff)
        self.w2 = nn.Linear(d_ff, d_model)
        self.dropout = nn.Dropout(p=dropput)

    def forward(self, x):
        return self.w2(self.dropout(F.relu(self.w1(x))))


# 函数调用
def dm_test_PositionwiseFeedForward():
    d_model = 512
    vocab = 10000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    my_embeddings = Embeddings(d_model, vocab)
    x = my_embeddings(x)
    dropout = 0.1
    max_len = 60
    my_pe = PositionalEncoding(d_model, dropout, max_len)
    pe_result = my_pe(x)
    head = 8
    query = key = value = pe_result
    # 输入掩码张量mask
    mask = Variable(torch.zeros(8, 4, 4))
    my_mha = MultiHeadAttention(head, d_model, dropout)
    x = my_mha(query, key, value, mask)
    # 测试前馈全连接层
    my_PFF = PositionwiseFeedForward(d_model=512, d_ff=64, dropput=0.1)
    ff_result = my_PFF(x)
    print('x_result--->',ff_result.shape,'\n',ff_result)
dm_test_PositionwiseFeedForward()

4.7规范化层

1.规范化层的作用

它是所有深层网络都需要的标准网络层，因为随着网络层数的增加，通过多层的计算后参数可能出现过大或者过小的情况，这样可能导致学习过程出现异常，模型可能收敛的非常慢，因此都会在一定层数后接规范化层进行数值的规范化，使其特征数值在合理范围内

import torch.nn as nn
import torch
from torch.autograd import Variable
from 文本嵌入层 import Embeddings
from 注意力机制 import PositionalEncoding
from 多头注意力机制 import MultiHeadAttention
from 前馈全连接层 import PositionwiseFeedForward


class LayerNorm(nn.Module):
    def __init__(self, features, eps=1e-6):
        super(LayerNorm, self).__init__()
        self.a2 = nn.Parameter(torch.ones(features))
        self.b2 = nn.Parameter(torch.zeros(features))
        self.eps = eps

    def forward(self, x):
        mean = x.mean(-1, keepdims=True)
        std = x.std(-1, keepdims=True)
        # 对数据进行标准化转换
        y = self.a2 * (x - mean) / (std + self.eps) + self.b2
        return y


# 函数调用
def dm_test_LayerNorm():
    embedding_dim = 512
    vocab = 1000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    emb = Embeddings(embedding_dim, vocab)
    embr = emb(x)
    dropout = 0.2
    max_len = 60
    x = embr
    pe = PositionalEncoding(embedding_dim, dropout, max_len)
    pe_result = pe(x)
    query = key = value = pe_result
    # 调用验证
    d_ff = 64
    head = 8
    # 多头注意力机制的输出  作为前馈全连接层的输入
    mask = Variable(torch.zeros(8, 4, 4))
    mha = MultiHeadAttention(head, embedding_dim, dropout)
    mha_result = mha(query, key, value, mask)
    x = mha_result
    ff = PositionwiseFeedForward(embedding_dim, d_ff, dropout)
    ff_result = ff(x)
    features = d_model = 512
    eps = 1e-6
    x = ff_result
    ln = LayerNorm(features, eps)
    ln_result = ln(x)
    print('规范化层：',ln_result.shape,'\n',ln_result)

dm_test_LayerNorm()

4.8子层连接结构

概念：输入到子层以及规范化层的过程中还使用了残差连接（跳跃连接），因此我们把这一部分结构整体叫作子层连接，在每个编码器层中都有两个子层，这两个子层加上周围的连接结构就形成了两个子层连接结构

子层连接结构代码：

import torch
import torch.nn as nn
from 规范化层 import LayerNorm
from torch.autograd import Variable
from 文本嵌入层 import Embeddings
from 多头注意力机制 import PositionalEncoding
from 多头注意力机制 import MultiHeadAttention


class SublayerConnection(nn.Module):
    def __init__(self, size, dropout=0.1):
        super(SublayerConnection, self).__init__()
        # 定义norm层
        self.norm = LayerNorm(size)
        # 定义dropout
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, sublayer):
        myres = x + self.dropout(sublayer(self.norm(x)))
        return myres


# 函数调用
def dm_test_SublayerConnection():
    size = 512
    head = 8
    d_model = 512
    vocab = 1000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    emb = Embeddings(d_model, vocab)
    embr = emb(x)
    dropout = 0.2
    max_len = 60
    x = embr
    pe = PositionalEncoding(d_model, dropout, max_len)
    pe_result = pe(x)
    x = pe_result
    mask = Variable(torch.zeros(8, 4, 4))
    # 多头自注意力子层
    self_attn = MultiHeadAttention(head, d_model)
    sublayer = lambda x: self_attn(x, x, x, mask)
    # 子层链接结构
    sc = SublayerConnection(size, dropout)
    sc_result = sc(x, sublayer)
    print('sc_result.shape--->', sc_result.shape)
    print('sc_result-->', sc_result)
dm_test_SublayerConnection()

4.9编码器层

作用：作为编码器的组成单元，每个编码器层完成一次对输入的特征的提取过程，即编码过程

代码演示

import torch
import torch.nn as nn
from 子层连接结构 import SublayerConnection
from 多头注意力机制 import clones
from torch.autograd import Variable
from 文本嵌入层 import Embeddings
from 多头注意力机制 import PositionalEncoding
from 多头注意力机制 import MultiHeadAttention
from 前馈全连接层 import PositionwiseFeedForward


# 编码器层的代码分析
class EncoderLayer(nn.Module):
    def __init__(self, size, self_atten, feed_forward, dropout):
        super(EncoderLayer, self).__init__()
        # 实例化多头注意力对象
        self.self_atten = self_atten
        self.feed_forward = feed_forward
        self.size = size
        self.sublayer = clones(SublayerConnection(size, dropout), 2)

    def forward(self, x, mask):
        x = self.sublayer[0](x, lambda x: self.self_atten(x, x, x, mask))
        x = self.sublayer[1](x, self.feed_forward)
        return x


# 函数调用
def dm_test_EncoderLaayer():
    d_model = 512
    vocab = 1000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    emb = Embeddings(d_model, vocab)
    embr = emb(x)
    dropout = 0.2
    max_len = 60
    x = embr
    pe = PositionalEncoding(d_model, dropout, max_len)
    pe_result = pe(x)
    x = pe_result
    size = 512
    head = 8
    d_ff = 64
    self_attn = MultiHeadAttention(head, d_model)
    ff = PositionwiseFeedForward(d_model, d_ff, dropout)
    mask = Variable(torch.zeros(8, 4, 4))
    my_encoderlayer = EncoderLayer(size, self_attn, ff, dropout)
    el_result = my_encoderlayer(x, mask)
    print('el_result.shape', el_result.shape, el_result)
dm_test_EncoderLaayer()

4.10编码器

概念：编码器对输入进行指定的特征提取过程，也称为编码，由n个编码器层堆叠而成

代码演示

import copy

import torch
import torch.nn as nn
from 多头注意力机制 import clones
from 规范化层 import LayerNorm
from torch.autograd import Variable
from 文本嵌入层 import Embeddings
from 多头注意力机制 import PositionalEncoding
from 多头注意力机制 import MultiHeadAttention
from 前馈全连接层 import PositionwiseFeedForward
from 编码器层 import EncoderLayer


class Encoder(nn.Module):
    def __init__(self, layer, N):
        super(Encoder, self).__init__()
        self.layers = clones(layer, N)
        self.norm = LayerNorm(layer.size)

    def forward(self, x, mask):
        for layer in self.layers:
            x = layer(x, mask)
        return self.norm(x)


# 函数调用
def dm_test_Encoder():
    d_model = 512
    vocab = 1000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    emb = Embeddings(d_model, vocab)
    embr = emb(x)
    dropout = 0.2
    max_len = 60
    x = embr
    pe = PositionalEncoding(d_model, dropout, max_len)
    pe_result = pe(x)
    x = pe_result
    size = 512
    head = 8
    d_model = 512
    d_ff = 64
    c = copy.deepcopy
    attn = MultiHeadAttention(head, d_model)
    dropout = 0.2
    ff = PositionwiseFeedForward(d_model, d_ff, dropout)
    layer = EncoderLayer(size, c(attn), c(ff), dropout)
    N=6
    mask=Variable(torch.zeros(8,4,4))
    en=Encoder(layer,N)
    en_result=en(x,mask)
    print('en_result.shape--->',en_result.shape)
    print('en_result--->',en_result)
dm_test_Encoder()

5.解码器部分的实现

5.1解码器介绍

解码器部分：

由N个解码器层堆叠而成

每个解码器层由三个子层连接结构组成

第一个子层连接结构包括一个多头自注意力子层和规范化层以及一个残差连接

第二个子层连接结构包括一个多头注意力子层和规范化层以及一个残差连接

第三个子层连接结构包括一个前馈全连接子层和规范化层以及一个残差连接

5.2解码器层的作用

作为解码器的组成单元，每个解码器层根据给定的输入向目标方向进行特征提取操作，即解码过程

import torch
import torch.nn as nn
from 规范化层 import LayerNorm
from torch.autograd import Variable
from 文本嵌入层 import Embeddings
from 多头注意力机制 import PositionalEncoding
from 多头注意力机制 import MultiHeadAttention


class SublayerConnection(nn.Module):
    def __init__(self, size, dropout=0.1):
        super(SublayerConnection, self).__init__()
        # 定义norm层
        self.norm = LayerNorm(size)
        # 定义dropout
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, sublayer):
        myres = x + self.dropout(sublayer(self.norm(x)))
        return myres


# 函数调用
def dm_test_SublayerConnection():
    size = 512
    head = 8
    d_model = 512
    vocab = 1000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    emb = Embeddings(d_model, vocab)
    embr = emb(x)
    dropout = 0.2
    max_len = 60
    x = embr
    pe = PositionalEncoding(d_model, dropout, max_len)
    pe_result = pe(x)
    x = pe_result
    mask = Variable(torch.zeros(8, 4, 4))
    # 多头自注意力子层
    self_attn = MultiHeadAttention(head, d_model)
    sublayer = lambda x: self_attn(x, x, x, mask)
    # 子层链接结构
    sc = SublayerConnection(size, dropout)
    sc_result = sc(x, sublayer)
    print('sc_result.shape--->', sc_result.shape)
    print('sc_result-->', sc_result)
dm_test_SublayerConnection()

5.3解码器

解码器的作用：根据编码器的结果以及上一次预测的结果，对下一次可能出现的值进行特征表示

import copy

import torch
import torch.nn as nn
from 多头注意力机制 import clones
from 规范化层 import LayerNorm
from torch.autograd import Variable
from 文本嵌入层 import Embeddings
from 多头注意力机制 import PositionalEncoding
from 多头注意力机制 import MultiHeadAttention
from 前馈全连接层 import PositionwiseFeedForward
from 解码器层 import DecoderLayer
from 编码器 import dm_test_Encoder


# 解码器代码分析
class Decoder(nn.Module):
    def __init__(self, layer, N):
        super(Decoder, self).__init__()
        self.layers = clones(layer, N)
        self.norm = LayerNorm(layer.size)

    def forward(self, x, memory, source_mask, target_mask):
        for layer in self.layers:
            x = layer(x, memory, source_mask, target_mask)
        return self.norm(x)


def dm_test_Decoder():
    d_model = 512
    vocab = 1000
    x = Variable(torch.LongTensor([[100, 2, 421, 508], [491, 998, 1, 221]]))
    emb = Embeddings(d_model, vocab)
    embr = emb(x)
    dropout = 0.2
    max_len = 60
    x = embr
    pe = PositionalEncoding(d_model, dropout, max_len)
    pe_result = pe(x)
    x = pe_result
    size = 512
    d_model = 512
    head = 8
    d_ff = 64
    dropout = 0.2
    c = copy.deepcopy
    attn = MultiHeadAttention(head, d_model)
    ff = PositionwiseFeedForward(d_model, d_ff, dropout)
    layer = DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout)
    N = 6
    x = pe_result
    en_result = dm_test_Encoder()
    memory = en_result
    # 掩码对象
    mask = Variable(torch.zeros(8, 4, 4))
    # sorce掩码  target掩码
    source_mask = target_mask = mask
    # 创建解码器对象
    de = Decoder(layer, N)
    # 解码器对象 解码
    de_result = de(x,memory,source_mask,target_mask)
    print(de_result)
    print(de_result.shape)
dm_test_Decoder()

6.输出部分实现

输出部分包括线性层和Softmax层

线性层：通过对上一步的线性变换得到指定维度的输出，也就是变换维度的作用

Softmax层：使最后一维向量中的数字缩放到0-1的概率值范围之内，并满足它们的和为1

import torch
import torch.nn as nn
import torch.nn.functional as F


class Generator(nn.Module):
    def __init__(self, d_model, vocab_size):
        super(Generator, self).__init__()
        self.project = nn.Linear(d_model, vocab_size)

    def forward(self, x):
        x = F.log_softmax(self.project(x), dim=-1)
        return x
#函数的调用
d_model=512
vocab_size=1000
my_generator=Generator(d_model,vocab_size)
#准备模型数据
x=torch.randn(2,4,512)
#数据经过out层
gen_result=my_generator(x)
print('gen_result--->',gen_result,'\n',gen_result)

7.模型构建

1.编码器解码器结构的代码实现

import torch
import torch.nn as nn
from torch.autograd import Variable
from 编码器 import  Encoder as  en
from  解码器 import  Decoder as de
from 输出部分 import  Generator as gen


class EncoderDecoder(nn.Module):
    def __init__(self, encoder, decoder, source_embed, target_embed, generator):
        super(EncoderDecoder, self).__init__()
        # 将参数传入到类中
        self.encoder = encoder
        self.decoder = decoder
        self.src_embed = source_embed
        self.tgt_embed = target_embed
        self.generator = generator

    def forward(self, source, target, source_mask, target_mask):
        return self.generator(self.decode(self.encode(source, source_mask), source_mask, target, target_mask))

    def encode(self, source, source_mask):
        return self.encoder(self.src_embed(source), source_mask)

    def decode(self, memory, source_mask, target, target_mask):
        return self.decoder(self.tgt_embed(target), memory, source_mask, target_mask)


# 实例化参数
vocab_size = 1000
d_model = 512
encoder = en
decoder = de
source_embed = nn.Embedding(vocab_size, d_model)
target_embed = nn.Embedding(vocab_size, d_model)
generator = gen
# 输入参数
source = target = Variable([[100, 2, 421, 508], [491, 998, 1, 221]])
source_mask = target_mask = Variable(torch.zeros(8, 4, 4))
ed = EncoderDecoder(encoder, decoder, source_embed, target_embed, generator)
ed_result = ed(source, target, source_mask, target_mask)
print(ed_result)
print(ed_result.shape)

2.transformer模型构建过程的代码分析

# transfromer模型构建过程的代码分析
import copy

import torch
import  torch.nn as nn
from 多头注意力机制 import MultiHeadAttention
from 前馈全连接层 import PositionwiseFeedForward
from 多头注意力机制 import PositionalEncoding
from 编码器解码器结构的代码实现 import EncoderDecoder
from 文本嵌入层 import Embeddings
from 编码器 import Encoder
from 编码器层 import EncoderLayer
from 解码器 import Decoder
from 解码器层 import DecoderLayer
from  输出部分 import  Generator
from  torch.autograd import  Variable


def make_model(source_vocab, target_vocab, N=6, d_model=512, d_ff=2048, head=8, dropout=0.1):
    c = copy.deepcopy
    attn = MultiHeadAttention(head=8, embedding_dim=512, dropout=dropout)
    ff = PositionwiseFeedForward(d_model=d_model, d_ff=d_ff, dropput=dropout)
    position = PositionalEncoding(d_model=d_model, dropout=dropout)
    # 构建EncoderDecoder对象
    model = EncoderDecoder(
        Encoder(EncoderLayer(d_model,c(attn),c(ff),dropout),N),
        Decoder(DecoderLayer(d_model,c(attn),c(ff),dropout),N),
        nn.Sequential(Embeddings(d_model,source_vocab),c(position)),
        nn.Sequential(Embeddings(d_model,target_vocab),c(position)),
        Generator(d_model,target_vocab))
    for p in model.parameters():
         if p.dim()>1:
             nn.init.xavier_uniform(p)
    return  model
def dm_test_make_model():
    source_vocab=500
    target_vocab=1000
    N=6
    my_transfrom_modelobj=make_model(source_vocab,target_vocab,N=6,d_model=512,d_ff=2048,head=8,dropout=0.1)
    print(my_transfrom_modelobj)
    source=target=Variable(torch.LongTensor([[1, 2, 3, 8], [3, 4, 1, 8]]))
    source_mask=target_mask=Variable(torch.zeros(8,4,4))
    mydata=my_transfrom_modelobj(source,target,source_mask,target_mask)
    print('mydata.shape--->',mydata.shape)
    print('mydata--->',mydata)
dm_test_make_model()