查看nn.Embedding(word_nums, hid_dim)的均值和方差是否是0，1

import torch
import torch.nn as nn

torch.manual_seed(66)
torch.cuda.manual_seed(66)
words = nn.Embedding(100, 200)

w0 = torch.tensor(0)
w1 = torch.tensor(1)
w2 = torch.tensor(2)
w3 = torch.tensor(3)
w4 = torch.tensor(4)
w5 = torch.tensor(5)
w6 = torch.tensor(6)
w7 = torch.tensor(7)
w8 = torch.tensor(8)
w9 = torch.tensor(9)

a0 = words(w0)
a1 = words(w1)
a2 = words(w2)
a3 = words(w3)
a4 = words(w4)
a5 = words(w5)
a6 = words(w6)
a7 = words(w7)
a8 = words(w8)
a9 = words(w9)

# print(a)
# print(a.size())
u0 = a0.mean(-1, keepdim=True)
s0 = (a0 - u0).pow(2).mean(-1, keepdim=True)

u1 = a1.mean(-1, keepdim=True)
s1 = (a1 - u1).pow(2).mean(-1, keepdim=True)

u2 = a2.mean(-1, keepdim=True)
s2 = (a2 - u2).pow(2).mean(-1, keepdim=True)

u3 = a3.mean(-1, keepdim=True)
s3 = (a3 - u3).pow(2).mean(-1, keepdim=True)

u4 = a4.mean(-1, keepdim=True)
s4 = (a4 - u4).pow(2).mean(-1, keepdim=True)

u5 = a5.mean(-1, keepdim=True)
s5 = (a5 - u5).pow(2).mean(-1, keepdim=True)
print("a0")
print(u0)
print(s0)
print("a1")
print(u1)
print(s1)
print("a2")
print(u2)
print(s2)
print("a3")
print(u3)
print(s3)
print("a4")
print(u4)
print(s4)
print("a5")
print(u5)
print(s5)


'''
与层归一化进行对比
norm = nn.LayerNorm([200])
r0 = norm(a0)
u0_ = r0.mean(-1, keepdim=True)
s0_ = (r0 - u0_).pow(2).mean(-1, keepdim=True)
print(u0_)
print(s0_)



a0:
层归一化之前的结果:
tensor([-0.0089], grad_fn=<MeanBackward1>)
tensor([1.1853], grad_fn=<MeanBackward1>)
层归一化之后的结果:
tensor([1.6689e-08], grad_fn=<MeanBackward1>)
tensor([1.0000], grad_fn=<MeanBackward1>)

'''