调用XLnet模型训练好的词向量做W2V的方法如下:
1.pip install pytorch_transformers
2.下载预训练模型
3.如下Getw2v()代码块获取词向量
4. 使用词向量进行其他后续操作,如进行句子相似性判断,做命名实体识别等。以下代码是进行句子相似性判断的示例。
from pytorch_transformers import XLNetModel,XLNetConfig,XLNetTokenizer
import torch
import numpy as np
def Getw2v():
#mname 为模型路径,此处为pytorch版本的模型,下载地址:https://github.com/ymcui/Chinese-XLNet
mname = r'E:\pythonProgram\xlnet_w2v\src\Xlnet_model'
config = XLNetConfig.from_pretrained(mname + '/config.json')
vocab_file = mname + '/spiece.model'
tokenizer = XLNetTokenizer.from_pretrained(vocab_file,do_lower_case = True)
print("开始构造词表")
vocab = []
for i in range(tokenizer.vocab_size):
vocab.append(tokenizer._convert_id_to_token(i))
print("词表长度:",len(vocab))
print("开始加载模型。。。")
model = XLNetModel.from_pretrained(mname)
print("开始加载embbdedding。。。")
emb = model.word_embedding.weight.data
emb = emb.numpy()
# print(emb)
print(np.shape(emb))
return tokenizer,emb
from sklearn.metrics.pairwise import cosine_similarity
from collections import Counter
def Get_sim(tokenizer,emb,x,y):
# 使用余弦相似度来判断句子之间的相似度
tokens1 = tokenizer.tokenize(x)
tokens2 = tokenizer.tokenize(y)
if len(tokens1) == 0 or len(tokens2) == 0:
return 0
tokfreqs1 = Counter(tokens1)
tokfreqs2 = Counter(tokens2)
attr1_embs = [emb[tokenizer.convert_tokens_to_ids(
[token])[0]] for token in tokfreqs1]
attr2_embs = [emb[tokenizer.convert_tokens_to_ids(
[token])[0]] for token in tokfreqs2]
embedding1 = np.average(attr1_embs, axis=0).reshape(1, -1)
embedding2 = np.average(attr2_embs, axis=0).reshape(1, -1)
sim = cosine_similarity(embedding1, embedding2)[0][0]
# print(sim)
return sim
if __name__ == '__main__':
tokenizer, emb = Getw2v()
x = '我想吃苹果和香蕉'
y = '我想买苹果和菠萝'
sim = Get_sim(tokenizer, emb,x,y)
print(sim)