NLP之神经网络语言模型(NNLM)_Torch

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable

dtype = torch.FloatTensor

sentences = [ "i like dog", "i love coffee", "i hate milk"]

word_list = " ".join(sentences).split()
word_list = list(set(word_list))
word_dict = {w: i for i, w in enumerate(word_list)}
number_dict = {i: w for i, w in enumerate(word_list)}
n_class = len(word_dict) # number of Vocabulary

# NNLM Parameter
n_step = 2 # n-1 in paper
n_hidden = 2 # h in paper
m = 2 # m in paper

def make_batch(sentences):
    input_batch = []
    target_batch = []

    for sen in sentences:
        word = sen.split()
        input = [word_dict[n] for n in word[:-1]] # 写出前(n-1)个词所对应的的位置
        target = word_dict[word[-1]] #预测第n个词

        input_batch.append(input)
        target_batch.append(target)

    return input_batch, target_batch

make_batch(sentences)

([[5, 0], [5, 4], [5, 1]], [2, 6, 3])

input_batch = []
target_batch = []

# Model
class NNLM(nn.Module):
    def __init__(self):
        super(NNLM, self).__init__()
        self.C = nn.Embedding(n_class, m)
        self.H = nn.Parameter(torch.randn(n_step * m, n_hidden).type(dtype))
        self.W = nn.Parameter(torch.randn(n_step * m, n_class).type(dtype))
        self.d = nn.Parameter(torch.randn(n_hidden).type(dtype))
        self.U = nn.Parameter(torch.randn(n_hidden, n_class).type(dtype))
        self.b = nn.Parameter(torch.randn(n_class).type(dtype))

    def forward(self, X):
        X = self.C(X)
        X = X.view(-1, n_step * m) # [batch_size, n_step * n_class]
        tanh = torch.tanh(self.d + torch.mm(X, self.H)) # [batch_size, n_hidden]
        output = self.b + torch.mm(X, self.W) + torch.mm(tanh, self.U) # [batch_size, n_class]
        return output

model = NNLM()

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

input_batch, target_batch = make_batch(sentences)
input_batch = Variable(torch.LongTensor(input_batch))
target_batch = Variable(torch.LongTensor(target_batch))

model(input_batch).shape

torch.Size([3, 7])

input_batch.shape

torch.Size([3, 2])

target_batch.shape

torch.Size([3])

input_batch

tensor([[5, 0],
        [5, 4],
        [5, 1]])

model(input_batch)

tensor([[ 2.4777, -1.9615,  3.1095,  1.3201,  2.9772,  1.8934, -3.3565],
        [ 4.5358, -5.6496,  7.6550,  1.0730,  2.4828, -0.2332, -5.0569],
        [ 0.0928,  1.4060,  0.2988,  1.9056,  1.4285,  3.9681, -0.2240]],
       grad_fn=<AddBackward0>)

target_batch

tensor([2, 6, 3])

make_batch(sentences)

([[5, 0], [5, 4], [5, 1]], [2, 6, 3])

# Training
for epoch in range(5000):

    optimizer.zero_grad()
    output = model(input_batch)

    # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot)
    loss = criterion(output, target_batch)
    if (epoch + 1)%1000 == 0:
        print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))

    loss.backward()
    optimizer.step()

# Predict
predict = model(input_batch).data.max(1, keepdim=True)[1]

# Test
print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()])

Epoch: 1000 cost = 0.382346
Epoch: 2000 cost = 0.051073
Epoch: 3000 cost = 0.014534
Epoch: 4000 cost = 0.005861
Epoch: 5000 cost = 0.002755
[['i', 'like'], ['i', 'love'], ['i', 'hate']] -> ['dog', 'coffee', 'milk']

predict.shape

torch.Size([3, 1])

predict_1 = model(input_batch)

predict_1.shape

torch.Size([3, 7])

predict

tensor([[5],
        [3],
        [0]])

predict_1

tensor([[ 2.7264, -3.8278,  0.1669,  3.1027, -3.1502, 10.1232, -0.8033],
        [-1.4627, -6.5132, -0.9443,  6.8087, -4.8649, -0.4327, -3.6853],
        [ 6.9640, -3.6173, -0.7420, -2.6381, -4.1076, -0.5775, -3.9949]],
       grad_fn=<AddBackward0>)

# pad documents to a max length of 4 words
max_length = 4
padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
print(padded_docs)
# define the model
input = Input(shape=(4, ))
x = Embedding(vocab_size, 8, input_length=max_length)(input)
x = Flatten()(x)
x = Dense(1, activation='sigmoid')(x)
model = Model(inputs=input, outputs=x)
# compile the model
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
# summarize the model
print(model.summary())
# fit the model
model.fit(padded_docs, labels, epochs=50, verbose=0)
# evaluate the model
loss, accuracy = model.evaluate(padded_docs, labels, verbose=0)
print('Accuracy: %f' % (accuracy * 100))