2022.6.9科研记录：PackedSequence代码示例pytorch（StackOverflow&GitHub）

1.How do you use PyTorch PackedSequence in code?(如何在代码中使用 PyTorch PackedSequence？)

回答2：
其实不用自己考虑排序恢复问题，让torch.nn.utils.rnn.pack_padded_sequence函数完成所有工作，通过设置参数enforce_sorted=False。
然后返回的 PackedSequence 对象会在其 sorted_indices 和 unsorted_indicies 属性中携带排序相关信息，这些信息可以被后面的 nn.GRU 或 nn.LSTM 正确使用来恢复原来的索引顺序。
可运行代码示例：

import torch
from torch import nn
from torch.nn.utils.rnn import pad_sequence, pack_padded_sequence, pad_packed_sequence

data = [torch.tensor([1]),
        torch.tensor([2, 3, 4, 5]), 
        torch.tensor([6, 7]),
        torch.tensor([8, 9, 10])]
lengths = [d.size(0) for d in data]

padded_data = pad_sequence(data, batch_first=True, padding_value=0) 
embedding = nn.Embedding(20, 5, padding_idx=0)
embeded_data = embedding(padded_data)

packed_data = pack_padded_sequence(embeded_data, lengths, batch_first=True, enforce_sorted=False)
lstm = nn.LSTM(5, 5, batch_first=True)
o, (h, c) = lstm(packed_data)

# (h, c) is the needed final hidden and cell state, with index already restored correctly by LSTM.
# but o is a PackedSequence object, to restore to the original index:

unpacked_o, unpacked_lengths = pad_packed_sequence(o, batch_first=True)
# now unpacked_o, (h, c) is just like the normal output you expected from a lstm layer.

print(unpacked_o, unpacked_lengths)

我们得到 unpacked_o, unpacked_lengths 的输出，如下所示：

# output (unpacked_o, unpacked_lengths):
tensor([[[ 1.5230, -1.7530,  0.5462,  0.6078,  0.9440],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]],

        [[ 1.8888, -0.5465,  0.5404,  0.4132, -0.3266],
         [ 0.1657,  0.5875,  0.4556, -0.8858,  1.1443],
         [ 0.8957,  0.8676, -0.6614,  0.6751, -1.2377],
         [-1.8999,  2.8260,  0.1650, -0.6244,  1.0599]],

        [[ 0.0637,  0.3936, -0.4396, -0.2788,  0.1282],
         [ 0.5443,  0.7401,  1.0287, -0.1538, -0.2202],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]],

        [[-0.5008,  2.1262, -0.3623,  0.5864,  0.9871],
         [-0.6996, -0.3984,  0.4890, -0.8122, -1.0739],
         [ 0.3392,  1.1305, -0.6669,  0.5054, -1.7222],
         [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000]]],
       grad_fn=<IndexSelectBackward>) tensor([1, 4, 2, 3])

将其与原始数据和长度进行比较，我们可以发现排序 - 恢复问题已经得到了很好的解决。

回答1：
不是最漂亮的一段代码，但这是我在浏览 PyTorch 论坛和文档后收集的供我个人使用的代码。当然可以有更好的方法来处理排序 - 恢复部分，但我选择它在网络本身

class Encoder(nn.Module):
    def __init__(self, vocab_size, embedding_size, embedding_vectors=None, tune_embeddings=True, use_gru=True,
                 hidden_size=128, num_layers=1, bidrectional=True, dropout=0.6):
        super(Encoder, self).__init__()
        self.embed = nn.Embedding(vocab_size, embedding_size, padding_idx=0)
        self.embed.weight.requires_grad = tune_embeddings
        if embedding_vectors is not None:
            assert embedding_vectors.shape[0] == vocab_size and embedding_vectors.shape[1] == embedding_size
            self.embed.weight = nn.Parameter(torch.FloatTensor(embedding_vectors))
        cell = nn.GRU if use_gru else nn.LSTM
        self.rnn = cell(input_size=embedding_size, hidden_size=hidden_size, num_layers=num_layers,
                        batch_first=True, bidirectional=True, dropout=dropout)

    def forward(self, x, x_lengths):
        sorted_seq_lens, original_ordering = torch.sort(torch.LongTensor(x_lengths), dim=0, descending=True)
        ex = self.embed(x[original_ordering])
        pack = torch.nn.utils.rnn.pack_padded_sequence(ex, sorted_seq_lens.tolist(), batch_first=True)
        out, _ = self.rnn(pack)
        unpacked, unpacked_len = torch.nn.utils.rnn.pad_packed_sequence(out, batch_first=True)
        indices = Variable(torch.LongTensor(np.array(unpacked_len) - 1).view(-1, 1)
                                                                       .expand(unpacked.size(0), unpacked.size(2))
                                                                       .unsqueeze(1))
        last_encoded_states = unpacked.gather(dim=1, index=indices).squeeze(dim=1)
        scatter_indices = Variable(original_ordering.view(-1, 1).expand_as(last_encoded_states))
        encoded_reordered = last_encoded_states.clone().scatter_(dim=0, index=scatter_indices, src=last_encoded_states)
        return encoded_reordered

2.PackedSentence.ipynb

import numpy as np
import torch
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence

torch.cuda.is_available()

True

print(torch.__version__)

1.1.0

vocabulary_size = 5
words = range(vocabulary_size)

# sequences数据：数字表示字典中的索引
data = [[1, 2, 3], [0, 0, 1, 2, 4], [1, 0, 0, 4], [4, 4, 2]]
# sorted: key应用于element的function，reverse=True为降序
data = sorted(data, key=len, reverse=True)

这个代码不是使用函数的内置参数 从而进行sort的（如果想解pack的话最好还是用内置参数把？）

# 每个序列的长度
lengths = [len(seq) for seq in data]
# max_length
T = len(data[0])
# batch_size
B = len(data)

data

[[0, 0, 1, 2, 4], [1, 0, 0, 4], [1, 2, 3], [4, 4, 2]]

# sequence: 将此序列转为one-hot表示；T：序列的max-length；N：one-hot的dim
def seq2onehot(sequence, T, N):
    data = np.zeros((T, N))
    # data[[0,1,2,...,len], []] = 1
    data[np.array(range(len(sequence))), sequence] = 1
    return data

# 列表生成式：生成的是python list类型
data = [seq2onehot(seq, T, vocabulary_size) for seq in data]

data

tensor([[[1., 0., 0., 0., 0.],
         [1., 0., 0., 0., 0.],
         [0., 1., 0., 0., 0.],
         [0., 0., 1., 0., 0.],
         [0., 0., 0., 0., 1.]],

        [[0., 1., 0., 0., 0.],
         [1., 0., 0., 0., 0.],
         [1., 0., 0., 0., 0.],
         [0., 0., 0., 0., 1.],
         [0., 0., 0., 0., 0.]],

        [[0., 1., 0., 0., 0.],
         [0., 0., 1., 0., 0.],
         [0., 0., 0., 1., 0.],
         [0., 0., 0., 0., 0.],
         [0., 0., 0., 0., 0.]],

        [[0., 0., 0., 0., 1.],
         [0., 0., 0., 0., 1.],
         [0., 0., 1., 0., 0.],
         [0., 0., 0., 0., 0.],
         [0., 0., 0., 0., 0.]]], dtype=torch.float64)

# numpy to tensor
data = torch.from_numpy(np.array(data))

sequence = pack_padded_sequence(data, lengths, batch_first=True)

sequence
PackedSequence(data=tensor([[1., 0., 0., 0., 0.],
        [0., 1., 0., 0., 0.],
        [0., 1., 0., 0., 0.],
        [0., 0., 0., 0., 1.],
        [1., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0.],
        [0., 0., 1., 0., 0.],
        [0., 0., 0., 0., 1.],
        [0., 1., 0., 0., 0.],
        [1., 0., 0., 0., 0.],
        [0., 0., 0., 1., 0.],
        [0., 0., 1., 0., 0.],
        [0., 0., 1., 0., 0.],
        [0., 0., 0., 0., 1.],
        [0., 0., 0., 0., 1.]], dtype=torch.float64), batch_sizes=tensor([4, 4, 4, 2, 1]), sorted_indices=None, unsorted_indices=None)

这能直接清了[0., 0., 0., 0., 0.]？太强了吧

sequence.batch_sizes

tensor([4, 4, 4, 2, 1])

sequence.data.shape

torch.Size([15, 5])