1.How do you use PyTorch PackedSequence in code?(如何在代码中使用 PyTorch PackedSequence?)
回答2:
其实不用自己考虑排序恢复问题,让torch.nn.utils.rnn.pack_padded_sequence函数完成所有工作,通过设置参数enforce_sorted=False。
然后返回的 PackedSequence 对象会在其 sorted_indices 和 unsorted_indicies 属性中携带排序相关信息,这些信息可以被后面的 nn.GRU 或 nn.LSTM 正确使用来恢复原来的索引顺序。
可运行代码示例:
import torch
from torch import nn
from torch.nn.utils.rnn import pad_sequence, pack_padded_sequence, pad_packed_sequence
data = [torch.tensor([1]),
torch.tensor([2, 3, 4, 5]),
torch.tensor([6, 7]),
torch.tensor([8, 9, 10])]
lengths = [d.size(0) for d in data]
padded_data = pad_sequence(data, batch_first=True, padding_value=0)
embedding = nn.Embedding(20, 5, padding_idx=0)
embeded_data = embedding(padded_data)
packed_data = pack_padded_sequence(embeded_data, lengths, batch_first=True, enforce_sorted=False)
lstm = nn.LSTM(5, 5, batch_first=True)
o, (h, c) = lstm(packed_data)
# (h, c) is the needed final hidden and cell state, with index already restored correctly by LSTM.
# but o is a PackedSequence object, to restore to the original index:
unpacked_o, unpacked_lengths = pad_packed_sequence(o, batch_first=True)
# now unpacked_o, (h, c) is just like the normal output you expected from a lstm layer.
print(unpacked_o, unpacked_lengths)
我们得到 unpacked_o, unpacked_lengths 的输出,如下所示:
# output (unpacked_o, unpacked_lengths):
tensor([[[ 1.5230, -1.7530, 0.5462, 0.6078, 0.9440],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]],
[[ 1.8888, -0.5465, 0.5404, 0.4132, -0.3266],
[ 0.1657, 0.5875, 0.4556, -0.8858, 1.1443],
[ 0.8957, 0.8676, -0.6614, 0.6751, -1.2377],
[-1.8999, 2.8260, 0.1650, -0.6244, 1.0599]],
[[ 0.0637, 0.3936, -0.4396, -0.2788, 0.1282],
[ 0.5443, 0.7401, 1.0287, -0.1538, -0.2202],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]],
[[-0.5008, 2.1262, -0.3623, 0.5864, 0.9871],
[-0.6996, -0.3984, 0.4890, -0.8122, -1.0739],
[ 0.3392, 1.1305, -0.6669, 0.5054, -1.7222],
[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]]],
grad_fn=<IndexSelectBackward>) tensor([1, 4, 2, 3])
将其与原始数据和长度进行比较,我们可以发现排序 - 恢复问题已经得到了很好的解决。
回答1:
不是最漂亮的一段代码,但这是我在浏览 PyTorch 论坛和文档后收集的供我个人使用的代码。当然可以有更好的方法来处理排序 - 恢复部分,但我选择它在网络本身
class Encoder(nn.Module):
def __init__(self, vocab_size, embedding_size, embedding_vectors=None, tune_embeddings=True, use_gru=True,
hidden_size=128, num_layers=1, bidrectional=True, dropout=0.6):
super(Encoder, self).__init__()
self.embed = nn.Embedding(vocab_size, embedding_size, padding_idx=0)
self.embed.weight.requires_grad = tune_embeddings
if embedding_vectors is not None:
assert embedding_vectors.shape[0] == vocab_size and embedding_vectors.shape[1] == embedding_size
self.embed.weight = nn.Parameter(torch.FloatTensor(embedding_vectors))
cell = nn.GRU if use_gru else nn.LSTM
self.rnn = cell(input_size=embedding_size, hidden_size=hidden_size, num_layers=num_layers,
batch_first=True, bidirectional=True, dropout=dropout)
def forward(self, x, x_lengths):
sorted_seq_lens, original_ordering = torch.sort(torch.LongTensor(x_lengths), dim=0, descending=True)
ex = self.embed(x[original_ordering])
pack = torch.nn.utils.rnn.pack_padded_sequence(ex, sorted_seq_lens.tolist(), batch_first=True)
out, _ = self.rnn(pack)
unpacked, unpacked_len = torch.nn.utils.rnn.pad_packed_sequence(out, batch_first=True)
indices = Variable(torch.LongTensor(np.array(unpacked_len) - 1).view(-1, 1)
.expand(unpacked.size(0), unpacked.size(2))
.unsqueeze(1))
last_encoded_states = unpacked.gather(dim=1, index=indices).squeeze(dim=1)
scatter_indices = Variable(original_ordering.view(-1, 1).expand_as(last_encoded_states))
encoded_reordered = last_encoded_states.clone().scatter_(dim=0, index=scatter_indices, src=last_encoded_states)
return encoded_reordered
2.PackedSentence.ipynb
import numpy as np
import torch
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
torch.cuda.is_available()
True
print(torch.__version__)
1.1.0
vocabulary_size = 5
words = range(vocabulary_size)
# sequences数据:数字表示字典中的索引
data = [[1, 2, 3], [0, 0, 1, 2, 4], [1, 0, 0, 4], [4, 4, 2]]
# sorted: key应用于element的function,reverse=True为降序
data = sorted(data, key=len, reverse=True)
这个代码不是使用函数的内置参数 从而进行sort的(如果想解pack的话最好还是用内置参数把?)
# 每个序列的长度
lengths = [len(seq) for seq in data]
# max_length
T = len(data[0])
# batch_size
B = len(data)
data
[[0, 0, 1, 2, 4], [1, 0, 0, 4], [1, 2, 3], [4, 4, 2]]
# sequence: 将此序列转为one-hot表示;T:序列的max-length;N:one-hot的dim
def seq2onehot(sequence, T, N):
data = np.zeros((T, N))
# data[[0,1,2,...,len], []] = 1
data[np.array(range(len(sequence))), sequence] = 1
return data
# 列表生成式:生成的是python list类型
data = [seq2onehot(seq, T, vocabulary_size) for seq in data]
data
tensor([[[1., 0., 0., 0., 0.],
[1., 0., 0., 0., 0.],
[0., 1., 0., 0., 0.],
[0., 0., 1., 0., 0.],
[0., 0., 0., 0., 1.]],
[[0., 1., 0., 0., 0.],
[1., 0., 0., 0., 0.],
[1., 0., 0., 0., 0.],
[0., 0., 0., 0., 1.],
[0., 0., 0., 0., 0.]],
[[0., 1., 0., 0., 0.],
[0., 0., 1., 0., 0.],
[0., 0., 0., 1., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.]],
[[0., 0., 0., 0., 1.],
[0., 0., 0., 0., 1.],
[0., 0., 1., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.]]], dtype=torch.float64)
# numpy to tensor
data = torch.from_numpy(np.array(data))
sequence = pack_padded_sequence(data, lengths, batch_first=True)
sequence
PackedSequence(data=tensor([[1., 0., 0., 0., 0.],
[0., 1., 0., 0., 0.],
[0., 1., 0., 0., 0.],
[0., 0., 0., 0., 1.],
[1., 0., 0., 0., 0.],
[1., 0., 0., 0., 0.],
[0., 0., 1., 0., 0.],
[0., 0., 0., 0., 1.],
[0., 1., 0., 0., 0.],
[1., 0., 0., 0., 0.],
[0., 0., 0., 1., 0.],
[0., 0., 1., 0., 0.],
[0., 0., 1., 0., 0.],
[0., 0., 0., 0., 1.],
[0., 0., 0., 0., 1.]], dtype=torch.float64), batch_sizes=tensor([4, 4, 4, 2, 1]), sorted_indices=None, unsorted_indices=None)
这能直接清了[0., 0., 0., 0., 0.]?太强了吧
sequence.batch_sizes
tensor([4, 4, 4, 2, 1])
sequence.data.shape
torch.Size([15, 5])