PyTorch深度学习实践12——RNN基础

• RNN主要处理具有序列关系的输入数据
• RNNcell：每个RNNcell的输入不仅有当前样本xi,还有前一个cell运算得到的结果hi-1
• 为什么循环：因为RNNcell一直在循环使用，即权重参数一直在参与各个样本输入的运算
• RNNcell中的线性运算：
• 如果用RNNcell来写，需要自己写循环不断通过RNNcell：

import torch
"需要:初始化h0,输入序列"
batch_size = 1
input_size = 4
hidden_size = 2
seq_len = 3
 
cell = torch.nn.RNNCell(input_size=input_size, hidden_size =  hidden_size)
 
dataset = torch.randn(seq_len,batch_size,input_size) #构造输入序列
hidden = torch.zeros(batch_size, hidden_size) #构造全是0的隐层,即初始化h0
 
for idex,input in enumerate(dataset):
    print('='*20, idex, '='*20)
    print('Input size:',input.shape)
    hidden = cell(input,hidden)
    print('outputs size:',hidden.shape)
    print('hidden:', hidden)

• 如果直接用RNN来写，则不用自己写循环：

 

import torch
 
batch_size = 1
input_size = 4
hidden_size = 2
seq_len = 3
num_layers = 2
 
cell = torch.nn.RNN(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers)
inputs = torch.randn(seq_len, batch_size, input_size)
hidden = torch.zeros(num_layers, batch_size, hidden_size)
 
out,hidden = cell(inputs,hidden)
 
print('output size:', out.shape)
print('out:',out)
print('hidden size:',hidden.shape)
print('hidden:', hidden)

example 12-1：通过RNNCell学习序列“hello”→“ohlol”转换的规律

1. 构建字典（建立character和index之间的映射关系），通过indices编码得到one-hot向量
2. 确定RNN的输出：一个四维向量，再分类器来实现文本序列的重新排列

import torch
import torch.nn as nn
import torch.optim as optim

input_size = 4
hidden_size = 4
batch_size = 1

# prepare data
idx2char = ['e', 'h', 'l', 'o']
x_data = [1, 0, 2, 2, 3] # hello
y_data = [3, 1, 2, 3, 2] # ohlol

one_hot_lookup = [[1, 0, 0, 0],
                  [0, 1, 0, 0],
                  [0, 0, 1, 0],
                  [0, 0, 0, 1]] # 分别对应0，1，2，3即e,h,l,o

x_one_hot = [one_hot_lookup[x] for x in x_data]

inputs = torch.Tensor(x_one_hot).view(-1, batch_size, input_size) #-1即seqLen
labels = torch.LongTensor(y_data).view(-1, 1) #(seqLen,1)

# define model
class Model(nn.Module):
    def __init__(self, input_size, hidden_size, batch_size):
        super(Model, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.batch_size = batch_size
        self.rnncell = nn.RNNCell(input_size=self.input_size,
                                  hidden_size=self.hidden_size)

    def forward(self, input, hidden):
        hidden = self.rnncell(input, hidden)
        return hidden

    def init_hidden(self):
        return torch.zeros(self.batch_size, self.hidden_size)

model = Model(input_size, hidden_size, batch_size)

# loss & optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.1)

# training cycle
for epoch in range(15):
    loss = 0
    optimizer.zero_grad()
    hidden = model.init_hidden() # h0
    print('predicted string:', end='')
    for input, label in zip(inputs, labels):
        hidden = model(input, hidden)
        loss += criterion(hidden, label)
        _, idx = hidden.max(dim=1) # hidden是4维的，分别表示e,h,l,o的概率值
        print(idx2char[idx.item()], end='')

    loss.backward()
    optimizer.step()
    print(',epoch [%d/15] loss = %.4lf' % (epoch +1, loss.item()))

 example 12-1：通过RNN学习序列“hello”→“ohlol”转换的规律

import torch
import torch.nn as nn
import torch.optim as optim

input_size = 4
hidden_size = 4
batch_size = 1
num_layers = 1
seq_len = 5

# prepare data
idx2char = ['e', 'h', 'l', 'o']
x_data = [1, 0, 2, 2, 3] # hello
y_data = [3, 1, 2, 3, 2] # ohlol

one_hot_lookup = [[1, 0, 0, 0],
                  [0, 1, 0, 0],
                  [0, 0, 1, 0],
                  [0, 0, 0, 1]] # 分别对应0，1，2，3即e,h,l,o

x_one_hot = [one_hot_lookup[x] for x in x_data]

inputs = torch.Tensor(x_one_hot).view(seq_len, batch_size, input_size)
labels = torch.LongTensor(y_data)

# define model
class Model(nn.Module):
    def __init__(self, input_size, hidden_size, batch_size, num_layers):
        super(Model, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.batch_size = batch_size
        self.num_layers = num_layers
        self.RNN = nn.RNN(input_size=self.input_size, hidden_size=self.hidden_size, num_layers=self.num_layers)

    def forward(self, input):
        hidden = torch.zeros(self.num_layers, self.batch_size, self.input_size)
        out, _ = self.RNN(input, hidden)
        return out.view(-1,self.hidden_size) #rashpe out to (seq_len×batch_size, hiddensize)


model = Model(input_size, hidden_size, batch_size, num_layers)

# loss & optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.05)

# training cycle
for epoch in range(15):
    optimizer.zero_grad()
    outputs = model(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

    print('outputs:',outputs)
    _, idx = outputs.max(dim=1)
    idx = idx.data.numpy()  # reshape to numpy
    print('idx', idx)
    print('Pridected:', ''.join([idx2char[x] for x in idx]), end='')  # end是不自动换行，''.join是连接字符串数组
    print(',Epoch [%d/15] loss = %.3f' % (epoch + 1, loss.item()))

 Embedding：

One-hot矩阵是high-dimension、sparse，hardcoded的，通过Embedding将one-hot稀疏矩阵映射成低维、稠密的矩阵。

网络模型：Embedding Layer + RNN Cell + Linear Layer

import torch
import torch.nn as nn
import torch.optim as optim

num_class = 4 #4个类别，
input_size = 4 #输入维度
hidden_size = 8 #隐层输出维度，有8个隐层
embedding_size = 10 #嵌入到10维空间
num_layers = 2 #2层的RNN
batch_size = 1
seq_len = 5 #序列长度5


# prepare data
idx2char = ['e', 'h', 'l', 'o']
x_data = [[1, 0, 2, 2, 3]] #(batch, seq_len) list
y_data = [3, 1, 2, 3, 2] # ohlol

inputs = torch.LongTensor(x_data)
labels = torch.LongTensor(y_data)

# define model
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.emb = nn.Embedding(input_size, embedding_size)
        self.rnn = nn.RNN(input_size=embedding_size,
                          hidden_size=hidden_size,
                          num_layers=num_layers,
                          batch_first=True)
        self.fc = nn.Linear(hidden_size, num_class)

    def forward(self, x):
        hidden = torch.zeros(num_layers, x.size(0), hidden_size)
        x = self.emb(x)
        x, _ = self.rnn(x, hidden)
        x = self.fc(x)
        return x.view(-1, num_class)

model = Model()

# loss & optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.05)

# training cycle
for epoch in range(15):
    optimizer.zero_grad()
    outputs = model(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

    print('outputs:',outputs)
    _, idx = outputs.max(dim=1)
    idx = idx.data.numpy()  # reshape to numpy
    print('idx', idx)
    print('Pridected:', ''.join([idx2char[x] for x in idx]), end='')  # end是不自动换行，''.join是连接字符串数组
    print(',Epoch [%d/15] loss = %.3f' % (epoch + 1, loss.item()))