转自:http://rzcoding.blog.163.com/blog/static/2222810172013101895642665/
在文本处理中,比如商品评论挖掘,有时需要了解每个评论分别和商品的描述之间的相似度,以此衡量评论的客观性。
![Python 文本挖掘:使用gensim进行文本相似度计算 - rzcoding - Explore in Data](http://img.aiuxian.com/tech/000/000/828/107_975_b06.jpg)
第二步# 读取txt 文档中的每条评论并用itertools 的yield 方法存储起来(比起把所有数据存在数组中,使用itertools 的内存效率高,具体原理请google)
class MyCorpus(object):
def __iter__(self): for line in open(datapath): yield line.split() from gensim import corpora, models, similarities # 以下是把评论通过gensim 转化为tf-idf 形式,程序具体解释参见52nlp的博客或gensim官方文档 Corp = MyCorpus() dictionary = corpora.Dictionary(Corp) corpus = [dictionary.doc2bow(text) for text in Corp] #把所有评论转化为词包(bag of words) tfidf = models.TfidfModel(corpus) #使用tf-idf 模型得出该评论集的tf-idf 模型 corpus_tfidf = tfidf[corpus] #此处已经计算得出所有评论的tf-idf 值
#读取商品描述的txt 文档
q_file = open(querypath, 'r') query = q_file.readline() q_file.close()
vec_bow = dictionary.doc2bow(query.split()) #把商品描述转为词包 vec_tfidf = tfidf[vec_bow] #直接使用上面得出的tf-idf 模型即可得出商品描述的tf-idf 值
index = similarities.MatrixSimilarity(corpus_tfidf) #把所有评论做成索引 sims = index[vec_tfidf] #利用索引计算每一条评论和商品描述之间的相似度 similarity = list(sims) #把相似度存储成数组,以便写入txt 文档 sim_file = open(storepath, 'w') for i in similarity: sim_file.write(str(i)+'\n') #写入txt 时不要忘了编码 sim_file.close()
![Python 文本挖掘:使用gensim进行文本相似度计算 - rzcoding - Explore in Data](http://img.aiuxian.com/tech/000/000/828/108_6e5_23b.jpg)
#! /usr/bin/env python2.7 #coding=utf-8
import logging from gensim import corpora, models, similarities
def similarity(datapath, querypath, storepath): logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) class MyCorpus(object): def __iter__(self): for line in open(datapath): yield line.split() Corp = MyCorpus() dictionary = corpora.Dictionary(Corp) corpus = [dictionary.doc2bow(text) for text in Corp] tfidf = models.TfidfModel(corpus) corpus_tfidf = tfidf[corpus] q_file = open(querypath, 'r') query = q_file.readline() q_file.close() vec_bow = dictionary.doc2bow(query.split()) vec_tfidf = tfidf[vec_bow] index = similarities.MatrixSimilarity(corpus_tfidf) sims = index[vec_tfidf] similarity = list(sims) sim_file = open(storepath, 'w') for i in similarity: sim_file.write(str(i)+'\n') sim_file.close()