transformer中softmax为什么除根号d_k

输入值不能太大，太大如下，梯度消失 

import torch
from torch import nn
from torch.nn import functional as F
import numpy as np

#from torch.autograd import Variable
#from torchvision.models import resnet50
#import torchvision.transforms as T
#torch.set_grad_enabled(True)

np_arr = np.array([3.0, 1.0, -3.0], dtype=np.float)
y = torch.tensor(np_arr, requires_grad=True)

print("input: ", y)
p = F.softmax(y, dim=0)
print("probability: ", p)

# 注意计算导数时,loss只能是一个值的,不能是有多个分量的
# 但是可以对多个参数求偏导
# 因此，只能分别对p[0]、p[1]、p[2]对应的loss求导
# 计算p1对y0,y1,y2的导数，y0、y2是i!=j的情况求导，得到的是负值

#计算p0对y0,y1,y2的导数
#Specify retain_graph=True when calling backward the first time.
print(torch.autograd.grad(p[0], [y], retain_graph=True))
#计算p1对y0,y1,y2的导数
print(torch.autograd.grad(p[1], [y], retain_graph=True))
#计算p2对y0,y1,y2的导数
print(torch.autograd.grad(p[2], [y], retain_graph=True))


'''
input:  tensor([ 3.,  1., -3.], dtype=torch.float64, requires_grad=True)
probability:  tensor([0.8789, 0.1189, 0.0022], dtype=torch.float64,
       grad_fn=<SoftmaxBackward>)
(tensor([ 0.1065, -0.1045, -0.0019], dtype=torch.float64),)
(tensor([-0.1045,  0.1048, -0.0003], dtype=torch.float64),)
(tensor([-0.0019, -0.0003,  0.0022], dtype=torch.float64),)
'''

import torch
from torch import nn
from torch.nn import functional as F
import numpy as np
torch.set_printoptions(precision=10, sci_mode=False)

#from torch.autograd import Variable
#from torchvision.models import resnet50
#import torchvision.transforms as T
#torch.set_grad_enabled(True)

np_arr = np.array([30.0, 10.0, -30.0], dtype=np.float)
y = torch.tensor(np_arr, requires_grad=True)

print("input: ", y)
p = F.softmax(y, dim=0)
print("probability: ", p)

# 注意计算导数时,loss只能是一个值的,不能是有多个分量的
# 但是可以对多个参数求偏导
# 因此，只能分别对p[0]、p[1]、p[2]对应的loss求导
# 计算p1对y0,y1,y2的导数，y0、y2是i!=j的情况求导，得到的是负值

#计算p0对y0,y1,y2的导数
#Specify retain_graph=True when calling backward the first time.
print(torch.autograd.grad(p[0], [y], retain_graph=True))
#计算p1对y0,y1,y2的导数
print(torch.autograd.grad(p[1], [y], retain_graph=True))
#计算p2对y0,y1,y2的导数
print(torch.autograd.grad(p[2], [y], retain_graph=True))

'''
input:  tensor([ 30.,  10., -30.], dtype=torch.float64, requires_grad=True)
probability:  tensor([    0.9999999979,     0.0000000021,     0.0000000000],
       dtype=torch.float64, grad_fn=<SoftmaxBackward>)
(tensor([     0.0000000021,     -0.0000000021,     -0.0000000000],
       dtype=torch.float64),)
(tensor([    -0.0000000021,      0.0000000021,     -0.0000000000],
       dtype=torch.float64),)
(tensor([    -0.0000000000,     -0.0000000000,      0.0000000000],
       dtype=torch.float64),)
'''