介绍
假设 X t ∈ R n × d X_t∈\R^{n×d} Xt∈Rn×d 是时间步 t t t 的小批量输入, H t ∈ R n × h H_t∈\R^{n×h} Ht∈Rn×h 是该时间步的隐藏变量,则:
H t = ϕ ( X t W x h + H t − 1 W h h + b h ) H_t=ϕ(X_t W_{xh}+H_{t−1}W_{hh}+b_h) Ht=ϕ(XtWxh+Ht−1Whh+bh)
其中,
W
x
h
∈
R
d
×
h
W_{xh}∈\R^{d×h}
Wxh∈Rd×h ,
W
h
h
∈
R
h
×
h
W_{hh}∈\R^{h×h}
Whh∈Rh×h ,
b
h
∈
R
1
×
h
b_h∈\R^{1×h}
bh∈R1×h ,
ϕ
ϕ
ϕ 函数是非线性激活函数。由于引入了
H
t
−
1
W
h
h
H_{t−1}W_{hh}
Ht−1Whh ,
H
t
H_t
Ht 能够捕捉截至当前时间步的序列的历史信息,就像是神经网络当前时间步的状态或记忆一样。由于
H
t
H_t
Ht 的计算基于
H
t
−
1
H_{t−1}
Ht−1 ,上式的计算是循环的,使用循环计算的网络即循环神经网络(recurrent neural network)。
在时间步 t ,输出层的输出为:
O
t
=
H
t
W
h
q
+
b
q
O_t=H_tW_{hq}+b_q
Ot=HtWhq+bq
其中
W
h
q
∈
R
h
×
q
,
b
q
∈
R
1
×
q
W_{hq}∈\R^{h×q} , b_q∈\R^{1×q}
Whq∈Rh×q,bq∈R1×q 。
定义模型
使用Pytorch中的nn.RNN来构造循环神经网络。在本节中,我们主要关注nn.RNN的以下几个构造函数参数:
- input_size - The number of expected features in the input x
- hidden_size – The number of features in the hidden state h
- nonlinearity – The non-linearity to use. Can be either ‘tanh’ or ‘relu’. Default: ‘tanh’
- batch_first – If True, then the input and output tensors are provided as (batch_size, num_steps, input_size). Default: False
这里的batch_first决定了输入的形状,我们使用默认的参数False,对应的输入形状是 (num_steps, batch_size, input_size)。
forward函数的参数为:
- input of shape (num_steps, batch_size, input_size): tensor containing the features of the input sequence.
- h_0 of shape (num_layers * num_directions, batch_size, hidden_size): tensor containing the initial hidden state for each element in the batch. Defaults to zero if not provided. If the RNN is bidirectional, num_directions should be 2, else it should be 1.
forward函数的返回值是:
- output of shape (num_steps, batch_size, num_directions * hidden_size): tensor containing the output features (h_t) from the last layer of the RNN, for each t.
- h_n of shape (num_layers * num_directions, batch_size, hidden_size): tensor containing the hidden state for t = num_steps.
现在我们构造一个nn.RNN实例,并用一个简单的例子来看一下输出的形状。
rnn_layer = nn.RNN(input_size=vocab_size, hidden_size=num_hiddens)
num_steps, batch_size = 35, 2
X = torch.rand(num_steps, batch_size, vocab_size)
state = None
Y, state_new = rnn_layer(X, state)
print(Y.shape, state_new.shape)
torch.Size([35, 2, 256]) torch.Size([1, 2, 256])
我们定义一个完整的基于循环神经网络的语言模型。
class RNNModel(nn.Module):
def __init__(self, rnn_layer, vocab_size):
super(RNNModel, self).__init__()
self.rnn = rnn_layer
self.hidden_size = rnn_layer.hidden_size * (2 if rnn_layer.bidirectional else 1)
self.vocab_size = vocab_size
self.dense = nn.Linear(self.hidden_size, vocab_size)
def forward(self, inputs, state):
# inputs.shape: (batch_size, num_steps)
X = to_onehot(inputs, vocab_size)
X = torch.stack(X) # X.shape: (num_steps, batch_size, vocab_size)
hiddens, state = self.rnn(X, state)
hiddens = hiddens.view(-1, hiddens.shape[-1]) # hiddens.shape: (num_steps * batch_size, hidden_size)
output = self.dense(hiddens)
return output, state
类似的,我们需要实现一个预测函数,与前面的区别在于前向计算和初始化隐藏状态。
def predict_rnn_pytorch(prefix, num_chars, model, vocab_size, device, idx_to_char,
char_to_idx):
state = None
output = [char_to_idx[prefix[0]]] # output记录prefix加上预测的num_chars个字符
for t in range(num_chars + len(prefix) - 1):
X = torch.tensor([output[-1]], device=device).view(1, 1)
(Y, state) = model(X, state) # 前向计算不需要传入模型参数
if t < len(prefix) - 1:
output.append(char_to_idx[prefix[t + 1]])
else:
output.append(Y.argmax(dim=1).item())
return ''.join([idx_to_char[i] for i in output])
使用权重为随机值的模型来预测一次。
model = RNNModel(rnn_layer, vocab_size).to(device)
predict_rnn_pytorch('分开', 10, model, vocab_size, device, idx_to_char, char_to_idx)
输出: ‘分开胸呵以轮轮轮轮轮轮轮’
接下来实现训练函数,这里只使用了相邻采样。
def train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
corpus_indices, idx_to_char, char_to_idx,
num_epochs, num_steps, lr, clipping_theta,
batch_size, pred_period, pred_len, prefixes):
loss = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
model.to(device)
for epoch in range(num_epochs):
l_sum, n, start = 0.0, 0, time.time()
data_iter = d2l.data_iter_consecutive(corpus_indices, batch_size, num_steps, device) # 相邻采样
state = None
for X, Y in data_iter:
if state is not None:
# 使用detach函数从计算图分离隐藏状态
if isinstance (state, tuple): # LSTM, state:(h, c)
state[0].detach_()
state[1].detach_()
else:
state.detach_()
(output, state) = model(X, state) # output.shape: (num_steps * batch_size, vocab_size)
y = torch.flatten(Y.T)
l = loss(output, y.long())
optimizer.zero_grad()
l.backward()
grad_clipping(model.parameters(), clipping_theta, device)
optimizer.step()
l_sum += l.item() * y.shape[0]
n += y.shape[0]
if (epoch + 1) % pred_period == 0:
print('epoch %d, perplexity %f, time %.2f sec' % (
epoch + 1, math.exp(l_sum / n), time.time() - start))
for prefix in prefixes:
print(' -', predict_rnn_pytorch(prefix, pred_len, model,vocab_size, device, idx_to_char,char_to_idx))
训练模型
num_epochs, batch_size, lr, clipping_theta = 250, 32, 1e-3, 1e-2
pred_period, pred_len, prefixes = 50, 50, ['分开', '不分开']
train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
corpus_indices, idx_to_char, char_to_idx,
num_epochs, num_steps, lr, clipping_theta,
batch_size, pred_period, pred_len, prefixes)
以上感谢伯禹学习平台以及Datawhale组织