python 词向量_Python——腾讯词向量的预处理

Python——腾讯词向量的预处理

目标

：下载腾讯词向量，并对其进行预处理，输出字向量与对应的tokenizer。

腾讯词向量下载地址：

Tencent AI Lab Embedding Corpus for Chinese Words and Phrases

。解压后是一个约16G的txt文件，命名为：Tencent_AILab_ChineseEmbedding.txt。

Python代码：

#!/usr/bin/env python

# -*- coding:utf-8 -*-

"""

@Time :2020/2/3

@Name :Zhang Wei

@File :tencent.py

@Software :Pycharm

"""

import pickle as pk

from gensim.models import KeyedVectors

from tqdm import tqdm

from keras.preprocessing.text import Tokenizer

from keras.preprocessing.sequence import pad_sequences

import numpy as np

# 加载pkl文件

def load_pkl(input_path):

with open(input_path, 'rb') as f:

loaded_obj = pk.load(f)

return loaded_obj

# 写入pkl文件

def to_pkl(content, output_path):

with open(output_path, 'wb') as f:

pk.dump(content, f)

# 加载腾讯词向量文件，清洗后，转存为tencent.txt

def load_tencent_word_embedding():

n = 0

with open('tencent.txt', 'a', encoding='utf-8', errors='ignore') as w_f:

with open('Tencent_AILab_ChineseEmbedding.txt', 'r', encoding='utf-8', errors='ignore')as f:

for i in tqdm(range(8824330)): # 似乎不同时期下载的词向量range并不一样

data = f.readline()

a = data.split()

if i == 0:

w_f.write('8748463 200\n') # 行数的写入也有可能不同

if len(a) == 201:

if not a[0].isdigit():

n = n + 1

w_f.write(data)

print(n) # 输出清洗后的range

model = KeyedVectors.load_word2vec_format('tencent.txt', binary=False, unicode_errors='ignore')

print("successfully load tencent word embedding!")

# 保存腾讯字向量以及对应的分词器

def save_charembedding(embedding_path, tokenizer_path):

flag, keras_embedding, words = 0, [], []

with open('tencent.txt','r',encoding='utf-8') as file:

for line in file:

flag += 1

if flag >= 3:

vectorlist = line.split() # 切分一行，分为词汇和词向量

if len(vectorlist[0]) == 1: # 单字: '\u4e00' <= vectorlist[0] <= '\u9fff'

vector = list(map(lambda x:float(x),vectorlist[1:])) # 对词向量进行处理

vec = np.array(vector) # 将列表转化为array

keras_embedding.append(vec)

words.append(vectorlist[0])

res = np.array(keras_embedding)

to_pkl(res, embedding_path) # 保存腾讯字向量

# 创建分词器Tokenzier对象

tokenizer = Tokenizer()

# fit_on_texts 方法

tokenizer.fit_on_texts(words)

to_pkl(tokenizer, tokenizer_path) # 保存腾讯字分词器

if __name__ == "__main__":

embedding_path = "keras_embedding.pkl"

tokenizer_path = "keras_tokenizer.pkl"

# save_charembedding(embedding_path, tokenizer_path) # 保存腾讯字向量与字分词器

tokenizer = load_pkl(tokenizer_path)

# 单元测试

query = "武汉加油。中国加油。"

text = " ".join(list(query)) # 对 "武汉加油" 进行切分,得到 "武 汉 加 油"

seq = tokenizer.texts_to_sequences([text])

print(query, seq)

输出：

武汉加油。中国加油。 [[1449, 1663, 304, 553, 96, 131, 451, 304, 553, 96]]

参考资料：