One-hot编码

import numpy as np

samples = ['The cat sat on the mat.', 'The dog ate my homework.']
token_index = {}
for sample in samples:
    # 在这里使用split()方法，在中文中需要分词
    for word in sample.split():
        if word not in token_index:
            # 给每一个词一个独有的索引值
            token_index[word] = len(token_index) + 1
print(token_index)
# We will only consider the first `max_length` words in each sample.
max_length = 10

# 将编码结果存在矩阵中
#初始化(句子数量)个矩阵，维度为（句子最大长度*单词数）的全0矩阵，然后再根据token_index将相应位置的值置为1
results = np.zeros((len(samples), max_length, max(token_index.values()) + 1))
for i, sample in enumerate(samples):
    for j, word in list(enumerate(sample.split()))[:max_length]:
        # get方法返回dict指定键的值
        index = token_index.get(word)
        results[i, j, index] = 1.
print(results)

当词太多时，token_index太大，提出hash方法,Index通过hash的方式确定

results = np.zeros((len(samples), max_length, demension))
print(results.shape)
for i, sample in enumerate(samples):
    for j, word in list(enumerate(sample.split()))[:max_length]:
        # get方法返回dict指定键的值
        index = abs(hash(word))% demension
        results[i, j, index] = 1.
print(results)

Keras封装好的one-hot方法：

from keras.preprocessing.text import Tokenizer
import numpy as np
samples = ['The cat sat on the mat.', 'The dog ate my homework.']

tokenizer = Tokenizer(num_words=20)
tokenizer.fit_on_texts(samples)
sequences = tokenizer.texts_to_sequences(samples)

one_hot_results = tokenizer.texts_to_matrix(samples, mode='binary')
print(one_hot_results[0])
print(one_hot_results[1])

word_index = tokenizer.word_index
print(word_index)
print('Found %s unique tokens.' % len(word_index))