import torch
import torch.nn as nn
torch.manual_seed(66)
torch.cuda.manual_seed(66)
words = nn.Embedding(100, 200)
w0 = torch.tensor(0)
w1 = torch.tensor(1)
w2 = torch.tensor(2)
w3 = torch.tensor(3)
w4 = torch.tensor(4)
w5 = torch.tensor(5)
w6 = torch.tensor(6)
w7 = torch.tensor(7)
w8 = torch.tensor(8)
w9 = torch.tensor(9)
a0 = words(w0)
a1 = words(w1)
a2 = words(w2)
a3 = words(w3)
a4 = words(w4)
a5 = words(w5)
a6 = words(w6)
a7 = words(w7)
a8 = words(w8)
a9 = words(w9)
# print(a)
# print(a.size())
u0 = a0.mean(-1, keepdim=True)
s0 = (a0 - u0).pow(2).mean(-1, keepdim=True)
u1 = a1.mean(-1, keepdim=True)
s1 = (a1 - u1).pow(2).mean(-1, keepdim=True)
u2 = a2.mean(-1, keepdim=True)
s2 = (a2 - u2).pow(2).mean(-1, keepdim=True)
u3 = a3.mean(-1, keepdim=True)
s3 = (a3 - u3).pow(2).mean(-1, keepdim=True)
u4 = a4.mean(-1, keepdim=True)
s4 = (a4 - u4).pow(2).mean(-1, keepdim=True)
u5 = a5.mean(-1, keepdim=True)
s5 = (a5 - u5).pow(2).mean(-1, keepdim=True)
print("a0")
print(u0)
print(s0)
print("a1")
print(u1)
print(s1)
print("a2")
print(u2)
print(s2)
print("a3")
print(u3)
print(s3)
print("a4")
print(u4)
print(s4)
print("a5")
print(u5)
print(s5)
'''
与层归一化进行对比
norm = nn.LayerNorm([200])
r0 = norm(a0)
u0_ = r0.mean(-1, keepdim=True)
s0_ = (r0 - u0_).pow(2).mean(-1, keepdim=True)
print(u0_)
print(s0_)
a0:
层归一化之前的结果:
tensor([-0.0089], grad_fn=<MeanBackward1>)
tensor([1.1853], grad_fn=<MeanBackward1>)
层归一化之后的结果:
tensor([1.6689e-08], grad_fn=<MeanBackward1>)
tensor([1.0000], grad_fn=<MeanBackward1>)
'''
查看nn.Embedding(word_nums, hid_dim)的均值和方差是否是0,1
最新推荐文章于 2024-03-18 06:00:00 发布