调用预训练好的XLnet词向量

 

调用XLnet模型训练好的词向量做W2V的方法如下：
1.pip install pytorch_transformers
2.下载预训练模型
3.如下Getw2v()代码块获取词向量
4. 使用词向量进行其他后续操作，如进行句子相似性判断，做命名实体识别等。以下代码是进行句子相似性判断的示例。

from pytorch_transformers import XLNetModel,XLNetConfig,XLNetTokenizer
import torch
import numpy as np
def Getw2v():
    #mname 为模型路径，此处为pytorch版本的模型,下载地址：https://github.com/ymcui/Chinese-XLNet
    mname = r'E:\pythonProgram\xlnet_w2v\src\Xlnet_model'
    config = XLNetConfig.from_pretrained(mname + '/config.json')
    vocab_file = mname + '/spiece.model'
    tokenizer = XLNetTokenizer.from_pretrained(vocab_file,do_lower_case = True)
    print("开始构造词表")
    vocab = []
    for i in range(tokenizer.vocab_size):
        vocab.append(tokenizer._convert_id_to_token(i))
    print("词表长度：",len(vocab))
    print("开始加载模型。。。")
    model = XLNetModel.from_pretrained(mname)
    print("开始加载embbdedding。。。")
    emb = model.word_embedding.weight.data
    emb = emb.numpy()
    # print(emb)
    print(np.shape(emb))

    return tokenizer,emb


from sklearn.metrics.pairwise import cosine_similarity
from collections import Counter
def Get_sim(tokenizer,emb,x,y):
    # 使用余弦相似度来判断句子之间的相似度
    tokens1 = tokenizer.tokenize(x)
    tokens2 = tokenizer.tokenize(y)

    if len(tokens1) == 0 or len(tokens2) == 0:
        return 0

    tokfreqs1 = Counter(tokens1)
    tokfreqs2 = Counter(tokens2)
    attr1_embs = [emb[tokenizer.convert_tokens_to_ids(
        [token])[0]] for token in tokfreqs1]
    attr2_embs = [emb[tokenizer.convert_tokens_to_ids(
        [token])[0]] for token in tokfreqs2]

    embedding1 = np.average(attr1_embs, axis=0).reshape(1, -1)
    embedding2 = np.average(attr2_embs, axis=0).reshape(1, -1)

    sim = cosine_similarity(embedding1, embedding2)[0][0]
    # print(sim)
    return sim
if __name__ == '__main__':
    tokenizer, emb = Getw2v()
    x = '我想吃苹果和香蕉'
    y = '我想买苹果和菠萝'
    sim = Get_sim(tokenizer, emb,x,y)
    print(sim)