我真的太棒了吧
"""
---
title: Long Short-Term Memory (LSTM)
summary: A simple PyTorch implementation/tutorial of Long Short-Term Memory (LSTM) modules.
---
# Long Short-Term Memory (LSTM)
This is a [PyTorch](https://pytorch.org) implementation of Long Short-Term Memory.
"""
# Optional 的作用是可选类型,作用几乎和带默认值的参数等价。
# tuple 允许list, tuple, set, frozenset, deque, 或生成器并强制转换为元组
from typing import Optional, Tuple
import torch
from torch import nn
from labml_helpers.module import Module
class LSTMCell(Module):
"""
## Long Short-Term Memory Cell
LSTM Cell computes $c$, and $h$. $c$ is like the long-term memory,
and $h$ is like the short term memory.
We use the input $x$ and $h$ to update the long term memory.
In the update, some features of $c$ are cleared with a forget gate $f$,
and some features $i$ are added through a gate $g$.
The new short term memory is the $\tanh$ of the long-term memory
multiplied by the output gate $o$.
Note that the cell doesn't look at long term memory $c$ when doing the update. It only modifies it.
Also $c$ never goes through a linear transformation.
This is what solves vanishing and exploding gradients.
Here's the update rule.
\begin{align}
c_t &= \sigma(f_t) \odot c_{t-1} + \sigma(i_t) \odot \tanh(g_t) \\
h_t &= \sigma(o_t) \odot \tanh(c_t)
\end{align}
$\odot$ stands for element-wise multiplication.
Intermediate values and gates are computed as linear transformations of the hidden
state and input.
\begin{align}
i_t &= lin_x^i(x_t) + lin_h^i(h_{t-1}) \\
f_t &= lin_x^f(x_t) + lin_h^f(h_{t-1}) \\
g_t &= lin_x^g(x_t) + lin_h^g(h_{t-1}) \\
o_t &= lin_x^o(x_t) + lin_h^o(h_{t-1})
\end{align}
"""
def __init__(self, input_size: int, hidden_size: int, layer_norm: bool = False):
super().__init__()
# These are the linear layer to transform the `input` and `hidden` vectors.
# One of them doesn't need a bias since we add the transformations.
# This combines $lin_x^i$, $lin_x^f$, $lin_x^g$, and $lin_x^o$ transformations.
# 我认为应该是输入的是input-size , hidden_size就等于输出的size
# self.hidden_lin = nn.Linear(hidden_size, 4 * hidden_size)
self.input_lin = nn.Linear(input_size, 4 * hidden_size, bias=False)
# This combines $lin_h^i$, $lin_h^f$, $lin_h^g$, and $lin_h^o$ transformations.
self.hidden_lin = nn.Linear(hidden_size, 4 * hidden_size)
# Whether to apply layer normalizations.
#
# Applying layer normalization gives better results.
# $i$, $f$, $g$ and $o$ embeddings are normalized and $c_t$ is normalized in
# $h_t = o_t \odot \tanh(\mathop{LN}(c_t))$
# 线性层是否规范 layer_norm 图层规范层是BooL
if layer_norm:
# 我们将我们需要的层放到一个集合,然后将这个集合放入到nn.MoudleList中,MoudleList没有forward
# 归一化的维度,我总共有四层,归一为hidden_size的维度
self.layer_norm = nn.ModuleList([nn.LayerNorm(hidden_size) for _ in range(4)])
# 归一层hidden_size的维度
self.layer_norm_c = nn.LayerNorm(hidden_size)
else:
# 如果不规范,nn.Identity不改变输入,直接return input,否则就输入什么就输出什么
self.layer_norm = nn.ModuleList([nn.Identity() for _ in range(4)])
self.layer_norm_c = nn.Identity()
def forward(self, x: torch.Tensor, h: torch.Tensor, c: torch.Tensor):
# We compute the linear transformations for $i_t$, $f_t$, $g_t$ and $o_t$
# using the same linear layers.
# 拼接? inpu_line是线性层,hidden_lin也是线性层,ifgo就是[3,24],24我能理解,但为什么是3呢?
ifgo = self.hidden_lin(h) + self.input_lin(x)
# Each layer produces an output of 4 times the `hidden_size` and we split them
# 按列分为4块,每一块是(3,6),batch_size是3
ifgo = ifgo.chunk(4, dim=-1)
# Apply layer normalization (not in original paper, but gives better results)
# 四个规范层
ifgo = [self.layer_norm[i](ifgo[i]) for i in range(4)]
# $$i_t, f_t, g_t, o_t$$ i为第一个3,6 f为第二个3,6 g为第3个 # ft为softmax(h[x],x[i])正确,i也是正确的,f正确,o呢,o也是一样的东西呀!!!g是什么,两个联合起来了就是c权重不一样,都是进行了一个拼接家伙是那个这个偏置
i, f, g, o = ifgo
# $$c_t = \sigma(f_t) \odot c_{t-1} + \sigma(i_t) \odot \tanh(g_t) $$ c_next 为3,6
# c_next 应该是ft*c+ i的softmax 之后乘以的是tanh(这个前面一样的玩意,
c_next = torch.sigmoid(f) * c + torch.sigmoid(i) * torch.tanh(g)# f是正确的,这里了softmax,之后乘以之前那的c,加上softmax这个i之后乘以这个tanh,这个g是什么,对正确
# $$h_t = \sigma(o_t) \odot \tanh(c_t)$$
# Optionally, apply layer norm to $c_t$ h_next为3,6
h_next = torch.sigmoid(o) * torch.tanh(self.layer_norm_c(c_next))# h_bext是一样的,输出,这个对的ot,之后乘以c_next,我对了!!!!
return h_next, c_next
class LSTM(Module):
"""
## Multilayer LSTM
"""
def __init__(self, input_size: int, hidden_size: int, n_layers: int):
"""
Create a network of `n_layers` of LSTM.
"""
super().__init__()
self.n_layers = n_layers
self.hidden_size = hidden_size
# Create cells for each layer. Note that only the first layer gets the input directly.
# Rest of the layers get the input from the layer below
self.cells = nn.ModuleList([LSTMCell(input_size, hidden_size)] +
[LSTMCell(hidden_size, hidden_size) for _ in range(n_layers - 1)])
def forward(self, x: torch.Tensor, state: Optional[Tuple[torch.Tensor, torch.Tensor]] = None):
"""
`x` has shape `[n_steps, batch_size, input_size]` and
`state` is a tuple of $h$ and $c$, each with a shape of `[batch_size, hidden_size]`.
"""
# x:tensor([[[ 0.3184, -0.7862, 0.5178, -0.2246, -0.2944],
# [ 2.4391, 1.0972, 0.0428, 0.8763, 1.2474],
# [ 0.4803, 1.6502, -0.2327, -0.1479, -1.4880]]])
n_steps, batch_size = x.shape[:2]# batch_size = 3 ,n_steps = 1
# Initialize the state if `None`
if state is None:
# h 和c 一样[tensor([[0., 0., 0., 0., 0., 0.],
# [0., 0., 0., 0., 0., 0.],
# [0., 0., 0., 0., 0., 0.]]), tensor([[0., 0., 0., 0., 0., 0.],
# [0., 0., 0., 0., 0., 0.],
# [0., 0., 0., 0., 0., 0.]])]
h = [x.new_zeros(batch_size, self.hidden_size) for _ in range(self.n_layers)]
c = [x.new_zeros(batch_size, self.hidden_size) for _ in range(self.n_layers)]
else:
(h, c) = state
# Reverse stack the tensors to get the states of each layer
#
# 📝 You can just work with the tensor itself but this is easier to debug
h, c = list(torch.unbind(h)), list(torch.unbind(c))
# Array to collect the outputs of the final layer at each time step.
out = []
for t in range(n_steps):
# Input to the first layer is the input itself
# inp = tensor[[3,5]], x 去掉一个大括号
inp = x[t]
# Loop through the layers layer先开始等于0,h[layer]
for layer in range(self.n_layers):
# Get the state of the layer h[0]为3,6 c[0]为3,6 next
h[layer], c[layer] = self.cells[layer](inp, h[layer], c[layer])
# Input to the next layer is the state of this layer
inp = h[layer] # inp为h[0],就是输入
# Collect the output $h$ of the final layer
out.append(h[-1]) # 第一次out 就是Inp,就是(3,6)
# Stack the outputs and states
out = torch.stack(out)# 维度的张量进行拼接 为(1,3,6)
h = torch.stack(h)# 为2,3,6)
c = torch.stack(c)# c之前是list=2 两个3,6,限制进行了拼接 变成了2,3,6
return out, (h, c)# h,c,inp=out是不一样的,好像是相加了友好也没有,不知道也,out为1,3,6 h和c为2,3,6
# inputline = inpuy_feature = 5 out_feature = 24
# hidden_size = 6, num_layer = 2
# hidden_lin in_feature = 6, output_feature= 24
LS = LSTM(5,6,2)
# tensor([[[ 0.3184],
# [ 2.4391],
# [ 0.4803]],
#
# [[-0.7862],
# [ 1.0972],
# [ 1.6502]],
#
# [[ 0.5178],
# [ 0.0428],
# [-0.2327]],
#
# [[-0.2246],
# [ 0.8763],
# [-0.1479]],
#
# [[-0.2944],
# [ 1.2474],
# [-1.4880]]])
# input 为 1:3:5
# x:tensor([[[ 0.3184, -0.7862, 0.5178, -0.2246, -0.2944],
# [ 2.4391, 1.0972, 0.0428, 0.8763, 1.2474],
# [ 0.4803, 1.6502, -0.2327, -0.1479, -1.4880]]])
input = torch.randn(1,3,5)
output , (hn,cn) = LS(input)
print(output)# 1 3 6 一模一样呀!!!
print("output:",output.shape)
print(hn)
print("hn:",hn.shape)# hn 2,3,6
print(cn)
print("cn:",cn.shape) # 2 3 6