# Layer Normalization解析

24 篇文章 161 订阅
53 篇文章 822 订阅

RuntimeError:
Given normalized_shape=[2],
expected input with shape [*, 2],
but got input of size[4, 2, 3]


import torch
import torch.nn as nn

def layer_norm_process(feature: torch.Tensor, beta=0., gamma=1., eps=1e-5):
var_mean = torch.var_mean(feature, dim=-1, unbiased=False)
# 均值
mean = var_mean[1]
# 方差
var = var_mean[0]

# layer norm process
feature = (feature - mean[..., None]) / torch.sqrt(var[..., None] + eps)
feature = feature * gamma + beta

return feature

def main():
t = torch.rand(4, 2, 3)
print(t)
# 仅在最后一个维度上做norm处理
norm = nn.LayerNorm(normalized_shape=t.shape[-1], eps=1e-5)
# 官方layer norm处理
t1 = norm(t)
# 自己实现的layer norm处理
t2 = layer_norm_process(t, eps=1e-5)
print("t1:\n", t1)
print("t2:\n", t2)

if __name__ == '__main__':
main()



# 仅在最后一个维度上做norm处理
norm = nn.LayerNorm(normalized_shape=t.shape[-1], eps=1e-5)


 tensor([[[-1.2758,  1.1659,  0.1099],
[ 0.6532, -1.4123,  0.7591]],

[[ 1.1400,  0.1522, -1.2922],
[ 1.0942, -1.3229,  0.2287]],

[[-0.9757, -0.3983,  1.3741],
[ 1.4134, -0.7379, -0.6755]],

[[ 0.1563,  1.1389, -1.2951],


 tensor([[[-1.2758,  1.1659,  0.1099],
[ 0.6532, -1.4123,  0.7591]],

[[ 1.1400,  0.1522, -1.2922],
[ 1.0942, -1.3229,  0.2287]],

[[-0.9757, -0.3983,  1.3741],
[ 1.4134, -0.7379, -0.6755]],

[[ 0.1563,  1.1389, -1.2951],
[-1.2341,  0.0203,  1.2138]]])


• 109
点赞
• 233
收藏
• 打赏
• 11
评论
09-12 253
10-08 758
07-21 2552
11-07 321
01-08 1667
07-23 9943
10-19 1万+
02-08 858
11-30 1314

### “相关推荐”对你有帮助么？

• 非常没帮助
• 没帮助
• 一般
• 有帮助
• 非常有帮助

¥2 ¥4 ¥6 ¥10 ¥20

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、C币套餐、付费专栏及课程。