目录
1.自注意力(self_attention)公式
2.代码实现
import math
import torch
from torch import nn
import dltools
num_hiddens, num_heads = 100, 5
dropout = 0.2
#使用多头注意力机制时, 让key_size, query_size, value_size都=num_hiddens
attention = dltools.MultiHeadAttention(num_hiddens, num_hiddens, num_hiddens, num_hiddens, num_heads, dropout)
#dropout与.eval()搭配使用
attention.eval()
MultiHeadAttention( (attention): DotProductAttention( (dropout): Dropout(p=0.2, inplace=False) ) (W_q): Linear(in_features=100, out_features=100, bias=False) (W_k): Linear(in_features=100, out_features=100, bias=False) (W_v): Linear(in_features=100, out_features=100, bias=False) (W_o): Linear(in_features=100, out_features=100, bias=False) )
batch_size, num_queries, valid_lens = 2, 4, torch.tensor([3, 2])
X = torch.ones((batch_size, num_queries, num_hiddens))
#创建不带位置信息的自注意力,就是让queries/keys/values都传X
attention(X, X, X, valid_lens).shape
torch.Size([2, 4, 100])
2.1位置编码的代码实现
#位置编码
class PositionalEncoding(nn.Module):
def __init__(self, num_hiddens, dropout, max_len=10000, **kwargs):
super().__init__(**kwargs)
self.dropout = nn.Dropout(dropout)
#创建一个存放位置编码的tensor
self.P = torch.zeros((1, max_len, num_hiddens)) #第0维度的1,便于后面与其他数据计算时进行广播机制
#除号左边的shape=(10000, 1),1在进行除法运算会进行广播机制,变成50
X = torch.arange(max_len, dtype=torch.float32).reshape(-1, 1) / torch.pow(10000, torch.arange(0, num_hiddens, 2, dtype=torch.float32)/num_hiddens)
#取出所有的偶数列, 进行赋值
self.P[:, :, 0::2] = torch.sin(X)
#取出所有的奇数列, 进行赋值
self.P[:, :, 1::2] = torch.cos(X)
#定义前向传播
def forward(self, X):
#: X.shape[1]表示只会索引到X的最大值
X = X + self.P[:, : X.shape[1], :].to(X.device)
return self.dropout(X)
encoding_dim, num_steps = 32, 60
#创建位置编码对象
pos_encoding = PositionalEncoding(encoding_dim, dropout=0)
pos_encoding.eval() #
X = pos_encoding(torch.zeros(1, num_steps, encoding_dim))
#调用pos_encoding中的self.P属性
P = pos_encoding.P[:, :X.shape[1], :]
X.shape, P.shape
(torch.Size([1, 60, 32]), torch.Size([1, 60, 32]))
#dltools绘图
dltools.plot(torch.arange(num_steps), P[0, :, 6:10].T, xlabel='Row (position)', figsize=(6, 2.5), legend=['Col %d' % d for d in torch.arange(6,10)])
3.知识点个人理解