NLP之word2vec,使用tensorflow2.0

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

sentences = [ "i like dog", "i like cat", "i like animal",
              "dog cat animal", "apple cat dog like", "dog fish milk like",
              "dog cat eyes like", "i like apple", "apple i hate",
              "apple i movie book music like", "cat dog hate", "cat dog like"]

word_sequence = " ".join(sentences).split()# 用空格分开,所有的单词 有重复单词

word_list = " ".join(sentences).split()
word_list = list(set(word_list)) #自动清除集合类型中的元素重复数据(set),以及元素排序
word_dict = {w: i for i, w in enumerate(word_list)} #编号,单词 : 编号

batch_size = 20  # To show 2 dim embedding graph
embedding_size = 2  # To show 2 dim embedding graph
voc_size = len(word_list)

voc_size

13

随机打乱词汇和标签

def random_batch(data, size):
    random_inputs = []
    random_labels = []
    random_index = np.random.choice(range(len(data)), size, replace=False)

    for i in random_index:
        random_inputs.append(np.eye(voc_size)[data[i][0]])  # target
        random_labels.append(data[i][1])  # context word

    return random_inputs, random_labels

跳词模型,由i-1,i+1个元素预测第i个元素

skip_grams = []
for i in range(1, len(word_sequence) - 1):
    target = word_dict[word_sequence[i]]
    context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]

    for w in context:
        skip_grams.append([target, w])

input_batch, target_batch = random_batch(skip_grams, batch_size)

input_batch = np.array(input_batch,dtype=np.float32)

target_batch = np.array(target_batch,dtype=np.float32)

input_batch #相当于独热编码

array([[0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],
       [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],
       [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.],
       [0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
       [0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.]],
      dtype=float32)

target_batch

array([ 4.,  8.,  4.,  2.,  8.,  2.,  5.,  7.,  7.,  5.,  6., 12.,  0.,
        5.,  6., 10.,  7.,  5.,  8.,  7.], dtype=float32)

class Word2Vec(tf.keras.Model):
    def __init__(self):
        super(Word2Vec, self).__init__()
        
        self.W = tf.Variable(-2 *np.random.rand(voc_size, embedding_size) + 1,dtype=tf.float32)
        self.WT = tf.Variable(-2 *np.random.rand(embedding_size, voc_size) + 1,dtype=tf.float32)
    
    def call(self, X):
        hidden_layer = tf.matmul(X, self.W)
        output_layer = tf.matmul(hidden_layer, self.WT)
        return output_layer

model = Word2Vec()

optimizer = tf.optimizers.Adam()
model.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',
              metrics=['acc'])#记录参数,记录loss,正确率acc

output = model(input_batch)

history = model.fit(input_batch,target_batch,epochs=2000)

Train on 20 samples
Epoch 1/2000
20/20 [==============================] - 0s 23ms/sample - loss: 7.7670 - acc: 0.2000


20/20 [==============================] - 0s 199us/sample - loss: 2.0134 - acc: 0.1500
Epoch 2000/2000
20/20 [==============================] - 0s 149us/sample - loss: 2.0134 - acc: 0.1500

W, WT = model.variables

W.numpy()

array([[ 0.5491909 ,  0.32456282],
       [ 0.02269172, -0.15633678],
       [-0.5128589 , -0.61800694],
       [-0.586934  , -0.36205733],
       [ 0.78309083, -1.5570229 ],
       [-0.02121925, -0.02437454],
       [-0.38094357, -0.2672366 ],
       [ 0.14762457,  0.5481064 ],
       [-0.0187831 , -0.02268313],
       [-0.8660246 ,  1.4212924 ],
       [-0.62751836, -0.890098  ],
       [ 1.0268191 ,  1.6595033 ],
       [-0.4092964 , -0.38745242]], dtype=float32)

for i, label in enumerate(word_list):
    W, WT = model.variables
    x,y = float(W[i][0]), float(W[i][1])#第一列为x,第二列为y
    plt.scatter(x, y)
    plt.annotate(label, xy=(x, y), xytext=(6, 2), textcoords='offset points', ha='right', va='bottom')
plt.show()