运用Python+Word2vec实现文本相似度
已爬取的新闻存放在sqlsever
训练好的model存放在model_out_put_path
// An highlighted block
import pymssql
import pandas as pd
import jieba
#from gensim import corpora, models,similarities
#import numpy as np
#import nltk
from gensim.models.word2vec import Word2Vec
###连接sql sever并读取数据到dataframe
conn = pymssql.connect('WIN-HNdgrtgwt', 'sa', '666666', 'test2')
cursor = conn.cursor()
sql_test = 'select * from test'
cursor.execute(sql_test)
test = pd.read_sql(sql_test, conn)
chapter_test = pd.DataFrame(test)
####
stopwords=pd.read_csv("C:\\Users\\test\\stopwords.txt",names=['stopword'], encoding='utf-8')
stoplist = list(stopwords.stopword)
dict = 'C:\\Users\\test\\自定义词库.txt'
jieba.load_userdict(dict)
def m_cut(intxt):
return [w for w in jieba.cut(intxt) if w not in stoplist and len(w)>1]
####文档预处理,提取主题词
chaplist_Title = [m_cut(w) for w in chapter_test.DocTitle]
chaplist_content= [m_cut(w) for w in chapter_test.EntryDescription]
###载入模型(已经训练好的模型)#####
model_out_put_path = "E:\\mymodel\\word2vec.model"
model = Word2Vec.load(model_out_put_path)
vocab_set = set(model.wv.vocab.keys())
###清除不在词汇表中的词语
def clear_word_from_vocab(word_list,vocab):
new_word_list = []
for word in word_list:
if word in vocab:
new_word_list.append(word)
return new_word_list
test_word_list = [clear_word_from_vocab(w,vocab_set) for w in chaplist_content]
test_doc = '''自然语言处理NLP数据挖掘'''
test_list = m_cut(test_doc)
test_list_clean = clear_word_from_vocab(test_list,vocab_set)
rank = []
for candidate in test_word_list:
score = model.n_similarity(test_list_clean,candidate)
rank.append(score)
#sim_score2 = model.n_similarity(test_list_clean,test_word_list[30])
M=[]
for item in chapter_test['RSSID']:
M.append(item)
m = list(zip(rank,M))
m.sort(reverse=True)
结果输出为:
[(0.8544847, 43627),
(0.83251715, 43619),
(0.83251715, 43609),
(0.8219972, 43622),
(0.8213873, 43604),
(0.81848884, 43605),
(0.7927264, 43628),
(0.7922138, 43600),
(0.773991, 43613),
(0.7541189, 43610),
(0.7402773, 43596),
(0.7347706, 43620),
(0.7236652, 43626),
(0.7195827, 43662),
(0.7056815, 43658)]