运用Python+Word2vec实现文本相似度

// An highlighted block
import pymssql
import pandas as pd
import jieba
#from gensim import corpora, models,similarities
#import numpy as np
#import nltk
from gensim.models.word2vec import Word2Vec

###连接sql sever并读取数据到dataframe
conn = pymssql.connect('WIN-HNdgrtgwt', 'sa', '666666', 'test2')
cursor = conn.cursor()
sql_test = 'select * from test'
      
cursor.execute(sql_test)
test = pd.read_sql(sql_test, conn)
chapter_test = pd.DataFrame(test)
####
stopwords=pd.read_csv("C:\\Users\\test\\stopwords.txt",names=['stopword'], encoding='utf-8')
stoplist = list(stopwords.stopword)
dict = 'C:\\Users\\test\\自定义词库.txt'
jieba.load_userdict(dict)
def m_cut(intxt):
    return [w for w in jieba.cut(intxt) if w not in stoplist and len(w)>1]
####文档预处理，提取主题词 
chaplist_Title = [m_cut(w) for w in chapter_test.DocTitle]
chaplist_content= [m_cut(w) for w in chapter_test.EntryDescription]
###载入模型（已经训练好的模型）#####
model_out_put_path = "E:\\mymodel\\word2vec.model"
model = Word2Vec.load(model_out_put_path)
vocab_set = set(model.wv.vocab.keys())
###清除不在词汇表中的词语
def clear_word_from_vocab(word_list,vocab):
    new_word_list = []
    for word in word_list:
        if word in vocab:
            new_word_list.append(word)
    return new_word_list
test_word_list = [clear_word_from_vocab(w,vocab_set) for w in chaplist_content]

test_doc = '''自然语言处理NLP数据挖掘'''
test_list = m_cut(test_doc)
test_list_clean = clear_word_from_vocab(test_list,vocab_set) 

rank = []
for candidate in test_word_list:
    score = model.n_similarity(test_list_clean,candidate)
    rank.append(score)
#sim_score2 = model.n_similarity(test_list_clean,test_word_list[30])      

M=[]
for item in chapter_test['RSSID']:
    M.append(item)
m = list(zip(rank,M))

m.sort(reverse=True)

结果输出为：

[(0.8544847, 43627),
 (0.83251715, 43619),
 (0.83251715, 43609),
 (0.8219972, 43622),
 (0.8213873, 43604),
 (0.81848884, 43605),
 (0.7927264, 43628),
 (0.7922138, 43600),
 (0.773991, 43613),
 (0.7541189, 43610),
 (0.7402773, 43596),
 (0.7347706, 43620),
 (0.7236652, 43626),
 (0.7195827, 43662),
 (0.7056815, 43658)]