Transformer 代码详细解析
谷歌在 《Attention is all you need》中提出了一种新模型的架构叫做Transformer,看了很多相关文章,通过图解能够大致理解模型的思路,但是依然无法深入理解本质,立足于代码无疑是深入理解的一个好方法。由于源代码中有较多变量名采用了缩写,并且变量的数据类型繁多,对于初学者可能依然难以理解,笔者在参照了源代码后进行了代码重写,并对每个变量名进行了后缀添加,同时添加了大量的注释。一方面深入自己的理解,另一方面同样也希望对初学者能够有所帮助。能力有限,难免会有很多不对的地方,不妥之处欢迎大家评论指出。
相关文章
1.《Attention is all you need》论文地址:
https://arxiv.org/abs/1706.03762
2.Transformer模型详解:
https://blog.csdn.net/u012526436/article/details/86295971(中文)
https://jalammar.github.io/illustrated-transformer/(英文)
3.源代码:
https://github.com/zingp/NLP/blob/master/P006TheAnnotatedTransformer/model_transformer.ipynb
完整代码
建议学习路径:中文图解模型→英文图解模型→论文原文→代码详解(本文)→源代码
"""
Copyright (C) 2019-2020 Dynamo
***************************************************
Author : Dynamo
Versions : 1.0
Time : 2020.1
File_name : Transformer 代码详细解析
***************************************************
# ↓ 导入pytorch相关模块
import torch
import torch.nn as nn # ↓ torch的主模块
import torch.nn.functional as TorchFunction_class
from torch.autograd import Variable
# ↓ 导入科学计算、画图等其它模块
import numpy as np
import matplotlib.pyplot as plt
import math, copy, time
import seaborn
seaborn.set_context(context="talk")
# ↓ ————————————————————————————————————定义Encoder-Decoder的类结构————————————————————————————————————
# ↓ 全局EncoderDecoder模型
class EncoderDecoder(nn.Module):
# ↓ 初始化网络,在实例化的时候(接受5个模块参数:编码模块,解码模块,词嵌入模块,目标嵌入模块,生成器模块)
def __init__(self, Encoder_class, Decoder_class, SourceEmbedding_class, TargetEmbedding_class, Generator_class):
super(EncoderDecoder, self).__init__()
print('Initial-EncoderDecoder')
# Encode层
self.Encoder_class = Encoder_class
# Decoder层
self.Decoder_class = Decoder_class
# 源语言和目标语言的embedding模块
self.SourceEmbedding_class = SourceEmbedding_class
self.TargetEmbedding_class = TargetEmbedding_class
# generator就是根据Decoder的隐状态输出当前时刻的词
# 基本的实现就是隐状态输入一个全连接层,全连接层的输出大小是词的个数
# 然后接一个softmax变成概率。
self.Generator_class = Generator_class
# ↓ 定义编码器
def Encoder_function(self, src, src_mask):
print('Forward-Encoder_function → ', end='')
print('InputData:', type(src), type(src_mask), end='')
OutPut_tensor = self.Encoder_class(self.SourceEmbedding_class(src), src_mask)
print('OutPutData:', type(OutPut_tensor))
print(OutPut_tensor)
return OutPut_tensor
# ↓ 定义解码器,decoder会比encoder多一个context attention
def Decoder_function(self, memory, src_mask, tgt, tgt_mask):
print('Forward-Decoder_function → ', end='')
print('InputData:', type(memory), type(src_mask), type(tgt), type(tgt_mask))
OutPut_tensor = self.Decoder_class(self.TargetEmbedding_class(tgt), memory, src_mask, tgt_mask)
return OutPut_tensor
# ↓ 向前传播(首先调用encode方法对输入进行编码,然后调用decode方法解码)
# OneBatch_tensor.InputSourceBatch_variable, OneBatch_tensor.InputTargetBatchIn_variable,
# OneBatch_tensor.InputSourceBatchMask_tensor, OneBatch_tensor.InputTargetBatchMask_tensor
def forward(self, OneBatchSourceBatch_variable, OneBatchTargetBatchIn_variable, OneBatchSourceBatchMask_variable,
OneBatchTargetBatchMask_variable):
print('Forward-EncoderDecoder → ', end='')
print('InputData:', type(OneBatchSourceBatch_variable), type(OneBatchTargetBatchIn_variable),
type(OneBatchSourceBatchMask_variable),
type(OneBatchTargetBatchMask_variable))
# (OneBatchSourceBatch, OneBatchSourceBatchMask) → Eecoder → MemoryEncoded_variable
MemoryEncoded_variable = self.Encoder_function(OneBatchSourceBatch_variable, OneBatchSourceBatchMask_variable)
# (MemoryEncoded_variable,SourceBatchMask,BatchTarget,TargetBatchMask) → Decoder → Decoded_variable
Decoded_variable = self.Decoder_function(MemoryEncoded_variable, OneBatchSourceBatchMask_variable,
OneBatchTargetBatchIn_variable, OneBatchTargetBatchMask_variable)
return Decoded_variable
# ↓ ————————————————————————————————————定义标准的线性+softmax生成步骤————————————————————————————————————
class Generator(nn.Module):
"""定义标准的线性+softmax生成步骤,这是在8. Embeddings和Softmax中"""
def __init__(self, Hyper_ModelDimension_int, Hyper_AllVocabNumber_int):
super(Generator, self).__init__()
print('Initial-Generator')
# ↓ 定义一层网络(词向量的维度数,输出的维度)
self.GeneratorLayer = nn.Linear(Hyper_ModelDimension_int, Hyper_AllVocabNumber_int)
def forward(self, Input_tensor):
print('Forward-Generator')
# ↓ 对这一层网络进行Softmax处理
Output_tensor = TorchFunction_class.log_softmax(self.GeneratorLayer(Input_tensor), dim=-1)
return Output_tensor
# ↓ ————————————————————————————————————定义Encoder和Decoder堆————————————————————————————————————
# ↓ 模型复制脚本
def CloneLayer_function(module, N):
# ↓ 生成N个相同的层。
return nn.ModuleList([copy.deepcopy(module) for Layer in range(N)])
# ↓ ————————————————————————————————————定义Encoder模型
class Encoder(nn.Module):
"""核心编码器是一个堆栈的N层"""
def __init__(self, EncodeLayer_class, N):
super(Encoder, self).__init__()
print('Initial-Encoder')
# ↓ 构造Encoder编码器,由N层堆叠而成,接收层模块
self.AllEncoderLayersList_class = CloneLayer_function(EncodeLayer_class, N) # 复制N层EncodeLayer
self.LayerNorm_class = LayerNorm(
EncodeLayer_class.Hyper_ModelDimension_int) # 层归一(接收参数为EncodeLayer层的ModelDimension)
def forward(self, Input_tensor, mask):
print('Forward-Encoder')
# ↓ for循环通过N个EncoderLayer
for SubEncoderLayer_class in self.AllEncoderLayersList_class:
Input_tensor = SubEncoderLayer_class(Input_tensor, mask)
# ↓ 层归一
OutPut_tensor = self.LayerNorm_class(Input_tensor)
return OutPut_tensor
# ↓ ————————————————————————————————————定义残差连接和
# ↓层归一
class LayerNorm(nn.Module):
def __init__(self, Hyper_ModelDimension_int, Hyper_MinimumConstant=1e-6):
super(LayerNorm, self).__init__()
print('Initial-LayerNorm')
# ↓返回一个全为1的张量,形状由Hyper_ModelDimension_int定义,为了弥补归一化的损失,这是一个训练参数
self.Weight_tensor = nn.Parameter(torch.ones(Hyper_ModelDimension_int))
# ↓返回一个全为0的张量,形状由Hyper_ModelDimension_int定义,为了弥补归一化的损失,这是一个训练参数
self.Bias_tensor = nn.Parameter(torch.zeros(Hyper_ModelDimension_int))
# ↓这是为了防止除以0而设置的一个极小的常量
self.Hyper_MinimumConstant = Hyper_MinimumConstant
def forward(self, Input_tensor):
print('Forward-LayerNorm')
# 1.按行求均值
# 2.按行求标准差
# 3.元素相乘(不是点积)
Mean = Input_tensor.mean(-1, keepdim=True)
StandardDeviation = Input_tensor.std(-1, keepdim=True)
OutPut_tensor = self.Weight_tensor * (Input_tensor - Mean) / (
StandardDeviation + self.Hyper_MinimumConstant) + self.Bias_tensor
return OutPut_tensor
# ↓ 残差连接
class SublayerConnection(nn.Module):
def __init__(self, Hyper_ModelDimension_int, Hyper_DropoutRate_float):
super(SublayerConnection, self).__init__()
print('Initial-SublayerConnection')
# 一层LayerNorm(x+Sublayer(x))。
self.LayerNorm_class = LayerNorm(Hyper_ModelDimension_int)
# 实例化一个Dropout
self.Dropout_class = nn.Dropout(Hyper_DropoutRate_float)
def forward(self, Input_tensor, InputSubLayer): # 将输入的tensor与这个tensor经过一个InputSubLayer层后的tensor相加
print('Forward-SublayerConnection')
# 1.首先进行层归一处理
# 2.再经过Sublayer
# 3.再进行Dropout处理
# 4.最后与处理前的tensor相加
OutPut_tensor = Input_tensor + self.Dropout_class(InputSubLayer(self.LayerNorm_class(Input_tensor)))
return OutPut_tensor
# ↓ ————————————————————————————————————定义EncoderLayer模型
class EncoderLayer(nn.Module):
"""编码器由self_attentions和前馈自调音组成(定义如下)
Encoder的每一层有两个block(教程中使用子层来称呼)"""
# EncoderLayer接收的参数:
# Hyper_ModelDimension_int, 模型维度
# copy.deepcopy(Copy_MultiHeadedAttention), 多头注意力机制模块
# copy.deepcopy(Copy_PositionwiseFeedForward), 位置信息模块
# Hyper_DropoutRate_float ,Dropout的比率
def __init__(self, Hyper_ModelDimension_int, Multi_HeadedAttention_class, PositionwiseFeedForward_class,
Hyper_DropoutRate_float):
super(EncoderLayer, self).__init__()
print('Initial-EncoderLayer')
# ↓ 多头注意机制
self.Multi_HeadedAttention_class = Multi_HeadedAttention_class
# ↓ FeedForward层
self.PositionwiseFeedForward_class = PositionwiseFeedForward_class
# 实例化并复制2个残差连接&归一化层,Sublayer的Forward要求传入两个参数(Input_tensor, Sublayer)
self.AllSublayerList_class = CloneLayer_function(
SublayerConnection(Hyper_ModelDimension_int, Hyper_DropoutRate_float), 2)
# ↓ 模型的维度(超参数)
self.Hyper_ModelDimension_int = Hyper_ModelDimension_int
def forward(self, Input_tensor, mask): # Input_tensor mask 是输入
print('Forward-EncoderLayer')
"""Follow Figure 1 (left) for connections."""
SubInput_tensor = self.AllSublayerList_class[0](Input_tensor,
lambda Input_tensor: self.Multi_HeadedAttention_class(
Input_tensor, Input_tensor, Input_tensor, mask))
OutPut_tensor = self.AllSublayerList_class[1](SubInput_tensor, self.PositionwiseFeedForward_class)
return OutPut_tensor
# ↓ ————————————————————————————————————构建Decoder堆
# ↓ Decoder模型
class Decoder(nn.Module):
"""Generic N layer decoder with masking."""
def __init__(self, DecodeLayer_class, N):
super(Decoder, self).__init__()
print('Initial-Decoder')
# ↓ 复制N层DecodeLayer
self.AllDecodeLayersList_class = CloneLayer_function(DecodeLayer_class, N)
# ↓ 在Decoder堆中每一层都要进行残差连接和层归一Layer-Normalization
self.LayerNorm_class = LayerNorm(DecodeLayer_class.Hyper_ModelDimension_int)
# ↓ 和encoder一样,复制N层,只是在每一层中比encoder多了memory。
def forward(self, Input_tensor, memory, src_mask, tgt_mask):
print('Forward-Decoder')
# ↓ for循环,tensor经过多层Decoder
for Decode_Layer in self.AllDecodeLayersList_class:
Input_tensor = Decode_Layer(Input_tensor, memory, src_mask, tgt_mask)
# ↓ 归一化
OutPut_tensor = self.LayerNorm_class(Input_tensor)
return OutPut_tensor
# ↓ Decoder层
class DecoderLayer(nn.Module):
"""解码器由self-attn、src-attn和前馈组成(定义如下)"""
# DecoderLayer接收的参数:
# Hyper_ModelDimension_int, 模型维度
# copy.deepcopy(Copy_MultiHeadedAttention), 多头注意力机制返回的tensor
# copy.deepcopy(Copy_MultiHeadedAttention), src_attn与多头注意力机制返回的tensor一样
# copy.deepcopy(Copy_PositionwiseFeedForward), 位置信息
# Hyper_DropoutRate_float, Dropout的比率
def __init__(self, Hyper_ModelDimension_int, MultiHeadedAttention_class, src_attn, PositionwiseFeedForward_class,
Hyper_DropoutRate_float):
super(DecoderLayer, self).__init__()
print('Initial-DecoderLayer')
# ↓ 模型的维度(超参数)
self.Hyper_ModelDimension_int = Hyper_ModelDimension_int
# ↓ 多头注意机制
self.MultiHeadedAttention_class = MultiHeadedAttention_class
# ↓ 多头注意机制
self.src_attn = src_attn
# ↓ FeedForward层
self.PositionwiseFeedForward_class = PositionwiseFeedForward_class
# ↓ 残差连接与归一化(3层)
self.AllSublayerList_class = CloneLayer_function(
SublayerConnection(Hyper_ModelDimension_int, Hyper_DropoutRate_float), 3)
def forward(self, Input_tensor, MemoryFromEncode_tensor, src_mask, tgt_mask):
print('Forward-DecoderLayer')
"""Follow Figure 1 (right) for connections."""
# 多头注意机制→残差连接与归一化→
# 多头注意机制→残差连接与归一化→
# FeedForward层→残差连接与归一化
MemoryFromEncode_tensor = MemoryFromEncode_tensor
Input_tensor = self.AllSublayerList_class[0](Input_tensor,
lambda Input_tensor: self.MultiHeadedAttention_class(Input_tensor,
Input_tensor,
Input_tensor,
tgt_mask))
Input_tensor = self.AllSublayerList_class[1](Input_tensor, lambda Input_tensor: self.src_attn(Input_tensor,
MemoryFromEncode_tensor,
MemoryFromEncode_tensor,
src_mask))
return self.AllSublayerList_class[2](Input_tensor, self.PositionwiseFeedForward_class)
# Decoder在解码第t个时刻的时候只能使用1-t时刻的输入,而不能使用t+1时刻及其之后的输入。因此我们需要一个函数来产生一个Mask矩阵
def CreateSubSequentMask_function(AttentionShape_matrix):
"""Mask out subsequent positions."""
AttentionShape = (1, AttentionShape_matrix, AttentionShape_matrix)
# np.triu产生一个上三角阵,k表示对角线起始位置,对角线下面的为0,上面的为1
SubSequentMask_np = np.triu(np.ones(AttentionShape), k=1).astype('uint8')
'''
例如:5*5的三角矩阵
[0, 1, 1, 1, 1],
[0, 0, 1, 1, 1],
[0, 0, 0, 1, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 0]
'''
# Numpy 的广播功能,判断matrix是否 = 0,返回一个0=True,else=False的矩阵
SubSequentMask_torch = torch.from_numpy(SubSequentMask_np) == 0
'''
例如:上述矩阵的变换
[ True, False, False, False, False],
[ True, True, False, False, False],
[ True, True, True, False, False],
[ True, True, True, True, False],
[ True, True, True, True, True]
'''
return SubSequentMask_torch
'''
打印三角矩阵
plt.figure(figsize=(5, 5))
plt.imshow(CreateSubSequentMask_function(20)[0])
None
'''
# ↓ ————————————————————————————————————Attention模块有四个维度(批数,头数,序列长度(句子长度),特征数(词向量))
# ↓ 注意力机制的计算脚本(query, key, value)它的输入是Query,Key,Value和Mask,输出是一个Tensor
def Attention_function(query, key, value, mask=None, dropout=None):
# input是维度为(批数,序列长度(句子长度),头数,特征数(词向量))
# 注意力机制的计算步骤:
# (1)查询向量*键向量(需要转置)
# (2)除以键向量维度的平方根
# (3)使用softmax函数标准化结果
# (4)将Score乘以Value,输出OutPut_tensor
# ↓ Key的维度
KeyDimension_int = query.size(-1)
# ↓ 查询向量*键向量(转置)query(B,T,H,F)*key(B,T,F,H)
Scores = torch.matmul(query, key.transpose(-2, -1))
# ↓ 除以键向量维度的平方根
Scores = Scores / math.sqrt(KeyDimension_int)
# ↓ 如果要启用Mask则把Mask是0的Scores位置变成一个很小的数(-1e9)
if mask is not None:
Scores = Scores.masked_fill(mask == 0, -1e9)
# ↓ 使用softmax函数处理结果获取Scores的权重
Product_Attention = TorchFunction_class.softmax(Scores, dim=-1)
# ↓ dropout处理
if dropout is not None:
Product_Attention = dropout(Product_Attention)
# ↓ 将Score乘以Value,最终输出OutPut_tensor
OutPut_tensor = torch.matmul(Product_Attention, value)
return OutPut_tensor, Product_Attention
# ↓ ————————————————————————————————————“多头”机制
class MultiHeadedAttention(nn.Module):
def __init__(self, Hyper_HeadNumber_int, Hyper_ModelDimension_int, Hyper_DropoutRate=0.1):
"""考虑模型大小和头的数量。"""
super(MultiHeadedAttention, self).__init__()
print('Initial-MultiHeadedAttention')
# 初始化时指定头数h(超参数)和模型维度Hyper_ModelDimension_int
assert Hyper_ModelDimension_int % Hyper_HeadNumber_int == 0 # 断言:二者是一定整除的,否则报错
# ↓ 一个MultiHeaded的大小(512/8=64)
self.HeadSize_int = Hyper_ModelDimension_int // Hyper_HeadNumber_int
# ↓ 一个MultiHeaded的个数
self.Hyper_HeadNumber_int = Hyper_HeadNumber_int
# ↓ 复制4层ModelDimension(特征数/词向量)维度的层,用于QKV做线性变换
self.AlllinearsList_class = CloneLayer_function(nn.Linear(Hyper_ModelDimension_int, Hyper_ModelDimension_int),
4)
self.Product_Attention = None
# ↓ 实例化一个Dropout方法
self.Hyper_DropoutRate = nn.Dropout(p=Hyper_DropoutRate)
def forward(self, InputQuery_tensor, InputKey_tensor, InputValue_tensor, mask=None):
print('Forward-MultiHeadedAttention')
# InputQuery_tensor有三个维度(批数(句子个数),序列长度(句子长度),特征数(词向量))
# ↓ "实现多头注意力模型",计算一下Mask的维度是(batch, 1, time),因为每个head的mask都是一样的,所以先用unsqueeze(1)变成(batch, 1, 1, time)
if mask is not None:
mask = mask.unsqueeze(1) # 在第1维后添加一个维度
# ↓ InputQuery_tensor有三个维度,这里提取第一个维度:批数
OneBatcheNumber_int = InputQuery_tensor.size(0)
# ↓ 1) 将这一批次的数据进行变形 d_model => h x d_k(列表生成器,将)
# 这里做了多步计算:
# zip(self.linears, (query, key, value))是把(self.linears[0],self.linears[1],self.linears[2])和(query, key, value)放到一起然后遍历。
# QKV + self.linears [层数] →
# QKV维度(OneBatcheNumber, OneSentenceNumber, OneWordNumber) + view →
# QKV维度:(OneBatcheNumber, OneSentenceNumber, 8, 64) + transpose(1, 2) →
# QKV维度(OneBatcheNumber, 8, OneSentenceNumber, 64)
InputQuery_tensor, InputKey_tensor, InputValue_tensor = \
[linear(InputQKV_tensor).view
(OneBatcheNumber_int, -1, self.Hyper_HeadNumber_int, self.HeadSize_int).transpose(1, 2)
# (OneBatcheNumber, time←→8, 64) → (OneBatcheNumber, 8, time, 64)
# ↑ 遍历4层ModelDimension*ModelDimension维度的层,做QKV经过4层线性变换
for linear, InputQKV_tensor in
zip(self.AlllinearsList_class, (InputQuery_tensor, InputKey_tensor, InputValue_tensor))]
# ↓ 2) 针对所有变量计算 Attention_function → Score(batch, 8, time, 64),Product_Attention(batch, 8, time, time)
Score, self.Product_Attention = Attention_function(InputQuery_tensor, InputKey_tensor, InputValue_tensor,
mask=mask,
dropout=self.Hyper_DropoutRate)
# ↓ 3) 最后,将Attention_function计算结果串联在一起
Score = Score.transpose(1, 2).contiguous().view(OneBatcheNumber_int, -1,
self.Hyper_HeadNumber_int * self.HeadSize_int)
return self.AlllinearsList_class[-1](Score)
# ↓ ————————————————————————————————————Position-wise前馈网络
# ↓ 在每一个block中,都包含有一个全连接的前馈神经网络。
# 包含两个线性变换,然后使用Relu作为激活函数。
class PositionwiseFeedForward(nn.Module):
"""Implements FFN equation."""
def __init__(self, Hyper_ModelDimension_int, Hyper_HiddenNodeNumber_int, Hyper_DropoutRate_float=0.1):
super(PositionwiseFeedForward, self).__init__()
print('Initial-PositionwiseFeedForward')
# ↓ 全连接层有两个线性变换
self.FeedForwardLayerOne = nn.Linear(Hyper_ModelDimension_int, Hyper_HiddenNodeNumber_int) # (512*512)
self.FeedForwardLayerTwo = nn.Linear(Hyper_HiddenNodeNumber_int, Hyper_ModelDimension_int)
self.dropout = nn.Dropout(Hyper_DropoutRate_float)
def forward(self, Input_tensor):
print('Forward-PositionwiseFeedForward')
# ↓ 线性变换
LayerOnePassed_tensor = self.FeedForwardLayerOne(Input_tensor)
# ↓ ReLU激活
ReLUed_tensor = TorchFunction_class.relu(LayerOnePassed_tensor)
# ↓ DropOut
DropOuted_tensor = self.dropout(ReLUed_tensor)
# ↓ 线性变换
Output_tensor = self.FeedForwardLayerTwo(DropOuted_tensor)
return Output_tensor
# ↓ Embeddings and Softmax
class Embeddings(nn.Module):
def __init__(self, Hyper_ModelDimension_int, Hyper_AllVocabNumber_int):
super(Embeddings, self).__init__()
print('Initial-Embeddings → ', end='')
print('EmbeddingsSize:' + str(Hyper_ModelDimension_int) + '*' + str(Hyper_AllVocabNumber_int))
# ↓词向量的维度
self.Hyper_ModelDimension_int = Hyper_ModelDimension_int
# ↓创建Hyper_VocabNumber_int个Hyper_ModelDimension_int维度的词向量查询矩阵
self.EmbeddingRequireMatrix_tensor = nn.Embedding(Hyper_AllVocabNumber_int, Hyper_ModelDimension_int)
def forward(self, InputRequireNumberInt_variable):
print('Forward-Embeddings')
# ↓ ModelDimension开方
ModelDimensionSquareRoot = math.sqrt(self.Hyper_ModelDimension_int)
# ↓ ModelDimension开方与Embedding_tensor相乘
print(InputRequireNumberInt_variable.shape)
Output_tensor = self.EmbeddingRequireMatrix_tensor(InputRequireNumberInt_variable) * ModelDimensionSquareRoot
return Output_tensor
# ↓ ————————————————————————————————————Positional encoding位置编码
# ↓ Positional Encoding
class PositionalEncoding(nn.Module):
"Implement the PE function."
def __init__(self, Hyper_ModelDimension_int, Hyper_DropoutRate, Hyper_MaxSentenceLength_int=5000):
super(PositionalEncoding, self).__init__()
print('Initial-PositionalEncoding')
# ↓ 实例化一个Dropout方法
self.Dropout = nn.Dropout(p=Hyper_DropoutRate)
# ↓ 创建PositionalEncoding矩阵(句子个数*词向量)
PositionalEncodingDimension_torch = torch.zeros(Hyper_MaxSentenceLength_int, Hyper_ModelDimension_int)
# ↓ 句子序列
ModelDimensionIndex_int = torch.arange(0.0, Hyper_MaxSentenceLength_int).unsqueeze(1)
# ↓ PositionalEncodingDimension的计算公式
PositionalEncodingDimension_torch[:, 0::2] = torch.sin(ModelDimensionIndex_int * torch.exp(
torch.arange(0.0, Hyper_ModelDimension_int, 2) * -(math.log(10000.0) / Hyper_ModelDimension_int))) # ↓ 偶数列
PositionalEncodingDimension_torch[:, 1::2] = torch.cos(ModelDimensionIndex_int * torch.exp(
torch.arange(0.0, Hyper_ModelDimension_int, 2) * -(math.log(10000.0) / Hyper_ModelDimension_int))) # ↓ 奇数列
# ↓ 添加一个维度
PositionalEncodingDimension_torch = PositionalEncodingDimension_torch.unsqueeze(0)
# ↓ register_buffer用于保存一些模型参数之外的值
self.register_buffer('PositionalEncoding', PositionalEncodingDimension_torch)
def forward(self, Input_tensor):
print('Forward-PositionalEncoding')
Input_tensor = Input_tensor + Variable(self.PositionalEncoding[:, :Input_tensor.size(1)],
requires_grad=False)
return self.Dropout(Input_tensor)
# ↓ 在位置编码下方,将基于位置添加正弦波。对于每个维度,波的频率和偏移都不同。
'''
plt.figure(figsize=(15, 5))
pe = PositionalEncoding(20, 0)
y = pe.forward(Variable(torch.zeros(1, 100, 20)))
plt.plot(np.arange(100), y[0, :, 4:8].data.numpy())
plt.legend(["dim %d" % p for p in [4, 5, 6, 7]])
None
'''
# ↓ ————————————————————————————————————构造一个简单的模型————————————————————————————————————
# ↓ 模型构造脚本
def CreateTransformerModel_function(Hyper_AllSourceVocabNumber_int, Hyper_AllTargetVocabNumber_int,
Hyper_NetLayer_int=6,
Hyper_ModelDimension_int=512,
Hyper_HiddenNodeNumber_int=2048,
Hyper_HeadNumber_int=8, Hyper_DropoutRate_float=0.1):
"""Helper: Construct a model from hyperparameters.从超参数构造模型。"""
Copy_MultiHeadedAttention = MultiHeadedAttention(Hyper_HeadNumber_int, Hyper_ModelDimension_int)
Copy_PositionwiseFeedForward = PositionwiseFeedForward(Hyper_ModelDimension_int, Hyper_HiddenNodeNumber_int,
Hyper_DropoutRate_float)
Copy_PositionalEncoding = PositionalEncoding(Hyper_ModelDimension_int, Hyper_DropoutRate_float)
model = EncoderDecoder(
Encoder(EncoderLayer(Hyper_ModelDimension_int, copy.deepcopy(Copy_MultiHeadedAttention),
copy.deepcopy(Copy_PositionwiseFeedForward), Hyper_DropoutRate_float), Hyper_NetLayer_int),
Decoder(DecoderLayer(Hyper_ModelDimension_int, copy.deepcopy(Copy_MultiHeadedAttention),
copy.deepcopy(Copy_MultiHeadedAttention), copy.deepcopy(Copy_PositionwiseFeedForward),
Hyper_DropoutRate_float), Hyper_NetLayer_int),
nn.Sequential(Embeddings(Hyper_ModelDimension_int, Hyper_AllSourceVocabNumber_int),
copy.deepcopy(Copy_PositionalEncoding)),
nn.Sequential(Embeddings(Hyper_ModelDimension_int, Hyper_AllTargetVocabNumber_int),
copy.deepcopy(Copy_PositionalEncoding)),
# ↓ 把模型的隐单元变成输出词的概率
Generator(Hyper_ModelDimension_int, Hyper_AllTargetVocabNumber_int)
)
# ↓ # 随机初始化参数,这非常重要
for Parameters in model.parameters():
if Parameters.dim() > 1:
nn.init.xavier_uniform_(Parameters)
return model
# ↓ 模型构造
# ↓ Small example model.
tmp_model = CreateTransformerModel_function(10, 10, 2) # src_vocab, tgt_vocab, Hyper_NetLayer_int=6
# ↓ ————————————————————————————————————训练
# ↓ Batches and Masking
class Batch:
"""Object for holding a batch of data with mask during training."""
def __init__(self, InputSourceBatch_variable, InputTargetBatch_variable=None, pad=0):
print('Initial-Batch →', end='')
print('Input:' + str(InputSourceBatch_variable.shape) + str(
InputTargetBatch_variable.shape)) # 30,10 30是大小,而10是最长的句子长度
# 输入为一个batch,包括多个句子
self.InputSourceBatch_variable = InputSourceBatch_variable
self.InputSourceBatchMask_variable = (InputSourceBatch_variable != pad).unsqueeze(-2)
if InputTargetBatch_variable is not None:
self.InputTargetBatchIn_variable = InputTargetBatch_variable[:, :-1]
self.InputTargetBatchOut_variable = InputTargetBatch_variable[:, 1:]
self.InputTargetBatchMask_variable = self.make_std_mask(self.InputTargetBatchIn_variable, pad)
self.ntokens = (self.InputTargetBatchOut_variable != pad).data.sum()
@staticmethod # 静态方法,可以不用实例化而调用这个函数,例如Batch.make_std_mask()
def make_std_mask(InputTargetBatch_tensor, pad):
print('Running-make_std_mask_function')
"""创建一个遮罩来隐藏填充和将来的单词。"""
InputTargetBatchMask_variable = (InputTargetBatch_tensor != pad).unsqueeze(-2)
InputTargetBatchMask_variable = InputTargetBatchMask_variable & Variable(
CreateSubSequentMask_function(InputTargetBatch_tensor.size(-1)).type_as(InputTargetBatchMask_variable.data))
return InputTargetBatchMask_variable
# ↓ Training Loop
def RunEpoch_function(AllData_Iterator, GlobalModel, LossComput_class):
"""Standard Training and Logging Function"""
StartTime_float = time.time()
total_tokens = 0
total_loss = 0
tokens = 0
for BatchIndex, OneBatch_tensor in enumerate(AllData_Iterator):
# 将数据传入全局模型得到输出(self, src, tgt, src_mask, tgt_mask):
GlobalModelOutput_tensor = GlobalModel.forward(OneBatch_tensor.InputSourceBatch_variable,
OneBatch_tensor.InputTargetBatchIn_variable,
OneBatch_tensor.InputSourceBatchMask_variable,
OneBatch_tensor.InputTargetBatchMask_variable)
loss = LossComput_class(GlobalModelOutput_tensor, OneBatch_tensor.InputTargetBatchOut_variable,
OneBatch_tensor.ntokens)
total_loss += loss
total_tokens += OneBatch_tensor.ntokens
tokens += OneBatch_tensor.ntokens
if BatchIndex % 50 == 1:
elapsed = time.time() - StartTime_float
print("Epoch Step: %d Loss: %f Tokens per Sec: %f" %
(BatchIndex, loss / OneBatch_tensor.ntokens, tokens / elapsed))
StartTime_float = time.time()
tokens = 0
return total_loss / total_tokens
# ↓ Training Data and Batching
global max_src_in_batch, max_tgt_in_batch
def batch_size_fn(new, count, sofar):
"Keep augmenting batch and calculate total number of tokens + padding."
global max_src_in_batch, max_tgt_in_batch
if count == 1:
max_src_in_batch = 0
max_tgt_in_batch = 0
max_src_in_batch = max(max_src_in_batch, len(new.src))
max_tgt_in_batch = max(max_tgt_in_batch, len(new.trg) + 2)
src_elements = count * max_src_in_batch
tgt_elements = count * max_tgt_in_batch
return max(src_elements, tgt_elements)
# ↓ Optimizer
class Optimizer:
"Optim wrapper that implements rate."
def __init__(self, model_size, factor, warmup, optimizer):
self.optimizer = optimizer
self._step = 0
self.warmup = warmup
self.factor = factor
self.model_size = model_size
self._rate = 0
def step(self):
"Update parameters and rate"
self._step += 1
rate = self.rate()
for p in self.optimizer.param_groups:
p['lr'] = rate
self._rate = rate
self.optimizer.step()
def rate(self, step=None):
"Implement `lrate` above"
if step is None:
step = self._step
return self.factor * \
(self.model_size ** (-0.5) *
min(step ** (-0.5), step * self.warmup ** (-1.5)))
def get_std_opt(model):
return Optimizer(model.src_embed[0].d_model, 2, 4000,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
'''
# ↓ Example of the curves of this model for different model sizes and for optimization hyperparameters.
# ↓ Three settings of the lrate hyperparameters.
opts = [Optimizer(512, 1, 4000, None),
Optimizer(512, 1, 8000, None),
Optimizer(256, 1, 4000, None)]
plt.plot(np.arange(1, 20000), [[opt.rate(i) for opt in opts] for i in range(1, 20000)])
plt.legend(["512:4000", "512:8000", "256:4000"])
None
'''
# ↓ Regularization
# ↓ Label Smoothing
class LabelSmoothing(nn.Module):
"Implement label smoothing."
def __init__(self, size, padding_idx, smoothing=0.0):
super(LabelSmoothing, self).__init__()
print('Initial-LabelSmoothing')
self.criterion = nn.KLDivLoss(reduction='sum')
self.padding_idx = padding_idx
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.size = size
self.true_dist = None
def forward(self, x, target):
print('Forward-LabelSmoothing')
assert x.size(1) == self.size
true_dist = x.data.clone()
true_dist.fill_(self.smoothing / (self.size - 2))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
true_dist[:, self.padding_idx] = 0
mask = torch.nonzero(target.data == self.padding_idx)
if mask.dim() > 0:
true_dist.index_fill_(0, mask.squeeze(), 0.0)
self.true_dist = true_dist
return self.criterion(x, Variable(true_dist, requires_grad=False))
############################################################第一个例子
# 给定一个小词汇表中的随机输入符号集,目标是生成这些相同的符号。
# 创造一些数据(这是一个迭代器)batch=30,nbatches=20
def CreateFakeData_function(Hyper_AllSourceVocabNumber_int, OneBatchNumber_int, AllBatchNumber_int):
print('Running:CreateFakeData_function')
# 一批的句子个数,一次构建一个句子的编码序列
for BatchNumber_int in range(AllBatchNumber_int):
# 返回一个范围1-Hyper_AllSourceVocabNumber_int(词序号)的随机整型数,size=一批的数量*10句的维度
SentenceData_torch = torch.from_numpy(
np.random.randint(1, Hyper_AllSourceVocabNumber_int, size=(OneBatchNumber_int, 10))).long()
SentenceData_torch[:, 0] = 1
SourceData_variable = Variable(SentenceData_torch, requires_grad=False).cuda()
TargetData_variable = Variable(SentenceData_torch, requires_grad=False).cuda()
# 实例化一个Batch类
CurrentBatchData_class = Batch(SourceData_variable, TargetData_variable, 0)
print('Output:CurrentBatchData_class:')
yield CurrentBatchData_class
# Loss Computation
class SimpleLossCompute:
"A simple loss compute and train function."
def __init__(self, generator, criterion, opt=None):
print('Initial-SimpleLossCompute')
self.generator = generator
self.criterion = criterion
self.opt = opt
def __call__(self, x, y, norm):
print('Call-SimpleLossCompute')
x = self.generator(x)
loss = self.criterion(x.contiguous().view(-1, x.size(-1)),
y.contiguous().view(-1)) / norm
loss.backward()
if self.opt is not None:
self.opt.step()
self.opt.optimizer.zero_grad()
OutputLoss = loss.item() * norm
return OutputLoss
# Greedy Decoding
# Train the simple copy task.
# 设置超参数一共11个字的字库
Hyper_AllSourceVocabNumber_int, Hyper_AllTargetVocabNumber_int = 10, 10
Hyper_NetLayer_int = 6
criterion = LabelSmoothing(size=Hyper_AllSourceVocabNumber_int, padding_idx=0, smoothing=0.0)
# 模型实例化11个源/标准字库,2层网络叠加
CopyTestModel_class = CreateTransformerModel_function(Hyper_AllSourceVocabNumber_int, Hyper_AllTargetVocabNumber_int,
Hyper_NetLayer_int).cuda()
# 定义优化器
CopyTestModelOptimizer = Optimizer(CopyTestModel_class.TargetEmbedding_class[0].Hyper_ModelDimension_int, 1, 400,
torch.optim.Adam(CopyTestModel_class.parameters(), lr=0, betas=(0.9, 0.98),
eps=1e-9))
# 训练多个Epoch
for Epoch in range(1):
# 指定训练和测试时采用不同方式,在训练模型时会在前面加上: model.train(启用Normalization 和 Dropout。)
CopyTestModel_class.train()
# 一共11个字的字库,每批30个句子,一共20批的生成器,一次生成一批
AllBatchData_Iterator = CreateFakeData_function(Hyper_AllSourceVocabNumber_int, 30, 20)
# 计算误差
LossCompute = SimpleLossCompute(CopyTestModel_class.Generator_class, criterion, CopyTestModelOptimizer)
# RunEpoch_function将训练一个Epoch的数据接受参数(AllData_Iterator, GlobalModel, loss_compute)
RunEpoch_function(AllBatchData_Iterator, CopyTestModel_class, LossCompute)
# model.train(不启用Normalization 和 Dropout。)
CopyTestModel_class.eval()
print(RunEpoch_function(CreateFakeData_function(Hyper_AllSourceVocabNumber_int, 30, 5), CopyTestModel_class,
SimpleLossCompute(CopyTestModel_class.Generator_class, criterion, None)))
结语
代码部分到此结束,第一次写博客,难免有各种纰漏,主要是记下自己的学习心得,通过重写代码笔者收益颇丰,同样希望读者通过阅读本文能够有所受益,最后,欢迎转载,但请注明出处。