transformer中的positional encoding(位置编码)计算理解

核心计算公式

原理bert-transform

代码理解

#pytorch
import torch
import math
max_len = 3
d_model = 4
pe = torch.zeros(3, 4)
position = torch.arange(0, max_len).unsqueeze(1)
div_term = torch.exp(torch.arange(0, d_model, 2) *
                             -(math.log(10000.0) / d_model))
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe

#sin编码
pe = np.zeros((3, 4))
i = 0
pos = 1
dmodel = 4
temp = 10000**(2*i/dmodel)
pe[pos,2*i] = math.sin(pos/temp)
pe

#cos编码
pe = np.zeros((3, 4))
i = 0
pos = 1
dmodel = 4
temp = 10000**(2*i/dmodel)
pe[pos,2*i+1] = math.cos(pos/temp)
pe