NLP FROM SCRATCH: CLASSIFY NAMES WITH A CHARACTER-LEVEL RNN
我们将建立和训练一个基于字符级的RNN模型，用来分类words。本教程将展示如何从零开始预处理数据，然后构建NLP模型。特别是没有使用torchtext的一些功能情况下，如何用底层模块进行NLP建模前的预处理工作。
基于字符级的RNN模型，以a series of characters的形式读取words，最终的预测的输出结果,是这个words属于哪一类。
具体来说，我们将18种语言的几千个姓氏进行训练，根据姓氏的拼写预测属于哪一类语言。如下展示：
$ python predict.py Hinton
(-0.47) Scottish
(-1.52) English
(-3.57) Irish
$ python predict.py Schmidhuber
(-0.19) German
(-2.48) Czech
(-2.68) Dutch
1. Preparing the Data
下载数据，路径：https://download.pytorch.org/tutorial/data.zip， 然后提取到当前目录。
目录data/names含有18个文件，以 “[Language].txt”命名。每个文件包含了许多姓氏(names)，每行一个姓氏(name)，很多事罗马字体书写。我们需要从Unicode编码转换到ASCII编码。
我们将会以字典形式表示，如：{language：[names,…]}。
from __future__ import unicode_literals, print_function, division
from io import open
import glob
import os
def findFiles(path):
return glob.glob(path)
print(findFiles('.data/data/names/*.txt'))
['.data/data/names\\Arabic.txt', '.data/data/names\\Chinese.txt', '.data/data/names\\Czech.txt', '.data/data/names\\Dutch.txt', '.data/data/names\\English.txt', '.data/data/names\\French.txt', '.data/data/names\\German.txt', '.data/data/names\\Greek.txt', '.data/data/names\\Irish.txt', '.data/data/names\\Italian.txt', '.data/data/names\\Japanese.txt', '.data/data/names\\Korean.txt', '.data/data/names\\Polish.txt', '.data/data/names\\Portuguese.txt', '.data/data/names\\Russian.txt', '.data/data/names\\Scottish.txt', '.data/data/names\\Spanish.txt', '.data/data/names\\Vietnamese.txt']
import unicodedata
import string
# string.ascii_letters返回值为 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
all_letters = string.ascii_letters + " .,;'"
n_letters = len(all_letters)
# turn a unicode string to ascii
def unicodeToAscii(s):
return ''.join(
c for c in unicodedata.normalize('NFD', s) # normalize() 第一个参数指定字符串标准化的方式
if unicodedata.category(c) != 'Mn' and c in all_letters # category()把一个字符返回它在UNICODE里分类的类型
)
print(unicodeToAscii('Ślusàrski'))
Slusarski
# Build the category_lines dictionary, a list of names per language
category_lines = {}
all_categories = []
# Read a file and split into lines
def readLines(filename):
lines = open(filename, encoding='utf-8').read().strip().split('\n')
return [unicodeToAscii(line) for line in lines]
for filename in findFiles('.data/data/names/*.txt'):
# # os.path.basename('.data/data/names\\Arabic.txt') 返回 'Arabic.txt'; os.path.splitext('Arabic.txt') 返回 ('Arabic', '.txt')
category = os.path.splitext(os.path.basename(filename))[0]
all_categories.append(category)
lines = readLines(filename)
category_lines[category] = lines
n_categories = len(all_categories)
print(category_lines['English'][:5])
['Abbas', 'Abbey', 'Abbott', 'Abdi', 'Abel']
2. Turning Names into Tensors
我们已经有了文本数据，下面需要把它们转换成Tensors，以便于使用它们。
我们使用one-hot vector进行字母表示，例如：“b”= <0, 1, 0, 0, …>，用n_letters表示vector的长度。因此构建单词，我们可以用2D矩阵表示 <line_length × n_letters>。由于pytorch的输入数据都是以batches形式，因此我们加入额外的1，即batches大小为1，即2D矩阵表示是<line_length × 1 × n_letters>
import torch
# Find letter index from all_letters, e.g. "a" = 0
def letterToIndex(letter):
return all_letters.find(letter)
# Just for demonstration, turn a letter into a <1 × n_letters> Tensor
def letterToTensor(letter):
tensor = torch.zeros(1, n_letters)
tensor[0][letterToIndex(letter)] = 1
return tensor
# Turn a line into a <line_length × 1 × n_letters>，
# or an array of one-hot letter vectors
def lineToTensor(line):
tensor = torch.zeros(len(line), 1, n_letters)
for li, letter in enumerate(line):
tensor[li][0][letterToIndex(letter)] = 1
return tensor
print(lineToTensor('zhang').size())
torch.Size([5, 1, 57])
3. Creating the Network
RNN原理图
公式如下
import torch.nn as nn
class RNN(nn.Module):
def __init__(self, input_size, hidden_size, output_size):
super(RNN, self).__init__()
self.hidden_size = hidden_size
self.i2h = nn.Linear(input_size + hidden_size, hidden_size)
self.i2o = nn.Linear(input_size + hidden_size, output_size)
# LogSoftmax其实就是对softmax的结果进行log，即Log(Softmax(x))
self.softmax = nn.LogSoftmax(dim=1)
def forward(self, input, hidden):
combined = torch.cat((input, hidden), 1) # 轴维度是1，进行拼接，比如：input: size([1, 57]), hidden: size([1, 128]), 拼接后combind: size([1, 185])
hidden = self.i2h(combined)
# print(hidden.size()) # torch.Size([1, 128])
output = self.i2o(combined)
# print(output.size()) # torch.Size([1, 18])
output = self.softmax(output)
# print(output.size()) # torch.Size([1, 18])
return output, hidden
def initHidden(self):
# 初始化初始时刻的hidden
return torch.zeros(1, self.hidden_size)
n_hidden = 128
rnn = RNN(n_letters, n_hidden, n_categories)
input = letterToTensor('A')
hidden = torch.zeros(1, n_hidden)
output, next_hidden = rnn(input, hidden)
input = lineToTensor('Albert')
hidden = torch.zeros(1, n_hidden)
print('input[0] size: ', input[0].size())
output, next_hidden = rnn(input[0], hidden)
print(output)
input[0] size: torch.Size([1, 57])
tensor([[-2.8574, -2.9403, -2.8987, -2.8663, -2.8174, -2.8526, -2.8784, -2.9954,
-2.8876, -2.9161, -2.8894, -2.9150, -2.8291, -2.8791, -2.9451, -2.9624,
-2.9141, -2.8041]], grad_fn=<LogSoftmaxBackward>)
4. Training
4.1 Preparing for Training
def categoryFromOutput(output):
# output.topk(k) 返回top k(从大到小排序)及其索引
top_n, top_i = output.topk(1)
category_i = top_i[0].item()
return all_categories[category_i], category_i
print(categoryFromOutput(output))
('Vietnamese', 17)
import random
def randomChoice(l):
# 随机选择一个category， random.randint(0, n)表示在0-n之间返回一个int。
return l[random.randint(0, len(l)-1)]
def randomTrainingExample():
# 随机某一个category
category = randomChoice(all_categories)
# 某category下的names，随机选择一个name
line = randomChoice(category_lines[category])
# 某category的索引值构建tensor
category_tensor = torch.tensor([all_categories.index(category)], dtype=torch.long)
# Turn a line into a <line_length × 1 × n_letters>，
line_tensor = lineToTensor(line)
return category, line, category_tensor, line_tensor
for i in range(10):
category, line, category_tensor, line_tensor = randomTrainingExample()
print('category = ', category, '/ line =', line)
category = Portuguese / line = Mata
category = Irish / line = Reynold
category = Greek / line = Patselas
category = Japanese / line = Tono
category = Dutch / line = Rijnders
category = Chinese / line = Weng
category = Korean / line = Hyun
category = Czech / line = Bacon
category = Vietnamese / line = Thao
category = Scottish / line = Hunter
4.2 Training the Network
由于RNN的最后一层是nn.LogSoftmax，所以损失函数用nn.NLLLoss是很合理的。
criterion = nn.NLLLoss()
learning_rate = 0.005
def train(category_tensor, line_tensor):
# 初始化第0时刻的hidden
hidden = rnn.initHidden()
rnn.zero_grad()
for i in range(line_tensor.size()[0]):
output, hidden = rnn(line_tensor[i], hidden)
# nn.NLLLoss()的输入，一个参数经Log(Softmax(x))产生的结果，如：[[a,b,c], [d, e, f]]，另一个参数是真实label，如：[2, 1]
# 那么返回结果：(abs(c) + abs(e))/2
loss = criterion(output, category_tensor)
loss.backward()
# 更新参数
for p in rnn.parameters():
p.data.add_(p.grad.data, alpha=-learning_rate)
return output, loss.item()
import time
import math
n_iters = 100000
print_every = 5000
plot_every = 1000
# keep track of losses for plotting
current_loss = 0
all_losses = []
def timeSince(since):
now = time.time()
s = now - since
m = math.floor(s / 60)
s -= m * 60
return '%dm %ds' % (m, s)
start = time.time()
for iter in range(1, n_iters + 1):
# 随机获取某category和name
category, line, category_tensor, line_tensor = randomTrainingExample()
output, loss = train(category_tensor, line_tensor)
current_loss += loss
if iter % print_every == 0:
# 返回：输出值 output 下的预测值和预测索引
guess, guess_i = categoryFromOutput(output)
correct = '✓' if guess == category else '✗ (%s)' % category
print('%d %d%% (%s) %.4f %s / %s %s' % (iter, iter / n_iters * 100, timeSince(start), loss, line, guess, correct))
if iter % plot_every == 0:
all_losses.append(current_loss / plot_every)
current_loss = 0
5000 5% (0m 7s) 3.4818 Gwang / English ✗ (Korean)
10000 10% (0m 14s) 2.7052 Pickard / Japanese ✗ (English)
15000 15% (0m 21s) 1.3434 Shum / Vietnamese ✗ (Chinese)
20000 20% (0m 28s) 2.8701 Duval / Arabic ✗ (French)
25000 25% (0m 36s) 0.2168 Dioletis / Greek ✓
30000 30% (0m 43s) 3.8010 See / Chinese ✗ (Dutch)
35000 35% (0m 50s) 1.6385 Rheem / Chinese ✗ (Korean)
40000 40% (0m 57s) 2.0398 Prince / French ✗ (English)
45000 45% (1m 5s) 0.2634 Lillis / Greek ✓
50000 50% (1m 12s) 2.4283 Seto / German ✗ (Chinese)
55000 55% (1m 19s) 1.3758 Rorris / Portuguese ✗ (Greek)
60000 60% (1m 26s) 0.6361 Gan / Chinese ✓
65000 65% (1m 34s) 2.3103 Bando / Italian ✗ (Japanese)
70000 70% (1m 41s) 1.2810 Chi / Vietnamese ✗ (Korean)
75000 75% (1m 48s) 2.4544 Nasato / Japanese ✗ (Italian)
80000 80% (1m 55s) 0.2323 Marchetti / Italian ✓
85000 85% (2m 3s) 0.0846 Sayuki / Japanese ✓
90000 90% (2m 10s) 1.3112 Han / Chinese ✗ (Vietnamese)
95000 95% (2m 17s) 0.6068 Suarez / Spanish ✓
100000 100% (2m 24s) 0.3425 Adamczyk / Polish ✓
4.3 Plotting the Results
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
plt.figure()
plt.plot(all_losses)
[<matplotlib.lines.Line2D at 0x2e3991a1a90>]
1.5 Evaluating the Results
# confusion matrix：每一行表示真实category，每一列表示预测的category
confusion = torch.zeros(n_categories, n_categories)
n_confusion = 10000
# Just return an output given a line
def evaluate(line_tensor):
hidden = rnn.initHidden()
for i in range(line_tensor.size()[0]):
output, hidden = rnn(line_tensor[i], hidden)
return output
# Go through a bunch of examples and record which are correctly guessed
for i in range(n_confusion):
category, line, category_tensor, line_tensor = randomTrainingExample()
output = evaluate(line_tensor)
guess, guess_i = categoryFromOutput(output)
category_i = all_categories.index(category)
# 表示真实是category_i下，预测guess_i的下个数。
confusion[category_i][guess_i] += 1
# Normalize by dividing every row by its sum
for i in range(n_categories):
# 每行标准化，也就是真实category的总量，同一行被分配的数量的占比
confusion[i] = confusion[i] / confusion[i].sum()
# set up plot
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(confusion.numpy())
fig.colorbar(cax)
# set up axes
ax.set_xticklabels([''] + all_categories, rotation=90)
ax.set_yticklabels([''] + all_categories)
# Force label at every tick
ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
ax.yaxis.set_major_locator(ticker.MultipleLocator(1))
# sphinx_gallery_thumbnail_number = 2
plt.show()
c:\users\86135\anaconda3\envs\torch\lib\site-packages\ipykernel_launcher.py:33: UserWarning: FixedFormatter should only be used together with FixedLocator
c:\users\86135\anaconda3\envs\torch\lib\site-packages\ipykernel_launcher.py:34: UserWarning: FixedFormatter should only be used together with FixedLocator
1.6 Running on User Input
def predict(input_line, n_predictions=3):
print('\n> %s' % input_line)
with torch.no_grad():
output = evaluate(lineToTensor(input_line))
# Get top N categories
topv, topi = output.topk(n_predictions, 1, True)
predictions = []
print('topv: ', topv)
print('topi: ', topi)
for i in range(n_predictions):
value = topv[0][i].item()
category_index = topi[0][i].item()
print('(%.2f) %s' % (value, all_categories[category_index]))
predictions.append([value, all_categories[category_index]])
predict('Dovesky')
predict('Jackson')
predict('Satoshi')
> Dovesky
topv: tensor([[-0.6312, -0.9500, -2.9799]])
topi: tensor([[14, 2, 4]])
(-0.63) Russian
(-0.95) Czech
(-2.98) English
> Jackson
topv: tensor([[-0.2220, -2.0125, -3.5026]])
topi: tensor([[15, 4, 14]])
(-0.22) Scottish
(-2.01) English
(-3.50) Russian
> Satoshi
topv: tensor([[-0.9452, -1.6715, -1.8808]])
topi: tensor([[ 9, 0, 10]])
(-0.95) Italian
(-1.67) Arabic
(-1.88) Japanese
基于以上代码，被划分为几个小文件，表示不同的功能实现，分别是：
data.py: loads filesmodel.py: defines the RNNtrain.py: runs trainingpredict.py: runs predict() with command line argumentsserver.py: serve prediction as a json api with bottle.py
Run train.py to train and save the network.
Run predict.py with a name to view predictions
网址：https://github.com/spro/practical-pytorch/tree/master/char-rnn-classification
扫一扫
1103
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
