李沐的经典著作《动手学深度学习》,推荐伯禹AI动手深度学习。
此系列为个人的学习和补充笔记,方便日后回顾。
一、SoftMax
推荐下面这篇博文,详细总结SoftMax。
二、RNN基础
用Pytorch版TextRNN,实现predict next step
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
dtype = torch.FloatTensor
sentences = [ "i like dog", "i love coffee", "i hate milk"]
word_list = " ".join(sentences).split()
word_list = list(set(word_list)) # 去重
print(word_list)
word_dict = {w: i for i, w in enumerate(word_list)}
number_dict = {i: w for i, w in enumerate(word_list)}
print(word_dict)
print(number_dict)
n_class = len(word_dict) # 词表大小,做n分类预测
# TextRNN Parameter
batch_size = len(sentences)
n_step = 2 # number of cells(= number of Step)
n_hidden = 5 # number of hidden units in one cell
def make_batch(sentences):
input_batch = []
target_batch = []
for sen in sentences:
word = sen.split()
input = [word_dict[n] for n in word[:-1]]
target = word_dict[word[-1]] # 最后一个单词作为target
print("input:", input, "traget:", target)
input_batch.append(np.eye(n_class)[input])
target_batch.append(target)
return input_batch, target_batch
# to Torch.Tensor
input_batch, target_batch = make_batch(sentences)
input_batch = Variable(torch.Tensor(input_batch))
target_batch = Variable(torch.LongTensor(target_batch))
print(input_batch)
print(target_batch) # [5 : dog, 0 : coffee, 1 : milk]
class TextRNN(nn.Module):
def __init__(self):
super(TextRNN, self).__init__()
self.rnn = nn.RNN(input_size=n_class, hidden_size=n_hidden)
self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype))
self.b = nn.Parameter(torch.randn([n_class]).type(dtype))
def forward(self, hidden, X):
X = X.transpose(0, 1) # X : [n_step, batch_size, n_class] [2, 3, 7]
outputs, hidden = self.rnn(X, hidden)
# outputs : [n_step, batch_size, num_directions(=1) * n_hidden]
# hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]
print("X : ",X ,"outputs : ",outputs ,"hidden : ",hidden)
outputs = outputs[-1] # [batch_size, num_directions(=1) * n_hidden]
model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class]
return model
model = TextRNN()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
# Training
for epoch in range(5000):
optimizer.zero_grad()
# hidden : [num_layers * num_directions, batch, hidden_size]
hidden = Variable(torch.zeros(1, batch_size, n_hidden))
# input_batch : [batch_size, n_step, n_class]
output = model(hidden, input_batch)
# output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot)
loss = criterion(output, target_batch)
if (epoch + 1) % 1000 == 0:
print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))
loss.backward()
optimizer.step()
Epoch: 1000 cost = 0.064772
Epoch: 2000 cost = 0.014707
Epoch: 3000 cost = 0.005852
Epoch: 4000 cost = 0.002837
Epoch: 5000 cost = 0.001509
input = [sen.split()[:2] for sen in sentences]
print(input)
# Predict
hidden = Variable(torch.zeros(1, batch_size, n_hidden))
predict = model(hidden, input_batch).data.max(1, keepdim=True)[1]
print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()])