gensim进行文本相似度比较两例

1.1 eg.1:

#-*- coding: utf-8 -*-
#example 1：
#将corpus以及query语料变换成bow向量，然后将bow向量变换成LSI主题模型向量，
#最后计算corpus的向量相对于query的向量的余弦相似度，并排序输出。
#reference: https://blog.csdn.net/l7h9ja4/article/details/80220939
from gensim import corpora,models,similarities

texts = [['human', 'interface', 'computer'],
['survey', 'user', 'computer', 'system', 'response', 'time'],
['eps', 'user', 'interface', 'system'],
['system', 'human', 'system', 'eps'],
['user', 'response', 'time'],
['trees'],
['graph', 'trees'],
['graph', 'minors', 'trees'],
['graph', 'minors', 'survey']]

dictionary = corpora.Dictionary(texts)

corpus = [dictionary.doc2bow(text) for text in texts]
query = dictionary.doc2bow(['system', 'human', 'system'])

lsi_model = models.LsiModel(corpus, id2word=dictionary, num_topics=2)
corpus_vec = lsi_model[corpus]
query_vec = lsi_model[query]

index = similarities.MatrixSimilarity(corpus_vec)
sims = index[query_vec] 

result = []
for i in range(len(sims)):
    result.append([sims[i], texts[i]])

result.sort(key=(lambda l: l[0]), reverse = True)
for i in result:
    print i

1.2 out:

[0.99965626, ['system', 'human', 'system', 'eps']]
[0.99711549, ['human', 'interface', 'computer']]
[0.99664235, ['eps', 'user', 'interface', 'system']]
[0.8808223, ['survey', 'user', 'computer', 'system', 'response', 'time']]
[0.84130645, ['user', 'response', 'time']]
[-0.087564662, ['graph', 'minors', 'survey']]
[-0.23449172, ['graph', 'minors', 'trees']]
[-0.24191013, ['graph', 'trees']]
[-0.25923383, ['trees']]

2.1 eg.2:

#-*- coding: utf-8 -*-
#example 2：
#将corpus以及query语料变换成bow向量，然后将bow向量变换成TF-IDF主题模型向量，
#最后计算corpus的向量相对于query的向量的余弦相似度，并排序输出。
#reference: https://blog.csdn.net/qq_42988748/article/details/82657562
from gensim import corpora,models,similarities

texts = [['human', 'interface', 'computer'],
['survey', 'user', 'computer', 'system', 'response', 'time'],
['eps', 'user', 'interface', 'system'],
['system', 'human', 'system', 'eps'],
['user', 'response', 'time'],
['trees'],
['graph', 'trees'],
['graph', 'minors', 'trees'],
['graph', 'minors', 'survey']]

dictionary = corpora.Dictionary(texts)
feature_cnt = len(dictionary.token2id.keys())

corpus = [dictionary.doc2bow(text) for text in texts]
query = dictionary.doc2bow(['system', 'human', 'system'])

tfidf = models.TfidfModel(corpus)
corpus_vec = tfidf[corpus]
query_vec = tfidf[query]

#index = similarities.MatrixSimilarity(corpus_vec, num_features=feature_cnt)
index = similarities.MatrixSimilarity(corpus_vec)
sims = index[query_vec] 

result = []
for i in range(len(sims)):
    result.append([sims[i], texts[i]])

result.sort(key=(lambda l: l[0]), reverse = True)
for i in result:
    print i

2.2 out:

[0.8706938, ['system', 'human', 'system', 'eps']]
[0.34416357, ['eps', 'user', 'interface', 'system']]
[0.32612574, ['human', 'interface', 'computer']]
[0.26776099, ['survey', 'user', 'computer', 'system', 'response', 'time']]
[0.0, ['user', 'response', 'time']]
[0.0, ['trees']]
[0.0, ['graph', 'trees']]
[0.0, ['graph', 'minors', 'trees']]
[0.0, ['graph', 'minors', 'survey']]