word2vec_文本相似度

#提取关键词
#关键词向量化
#相似度计算

from jieba import analyse
import numpy
import gensim


# 实现给出任意字符串，获取字符串中某字符的位置以及出现的总次数
def get_char_pos(string,char):
  chPos=[]
  try:
    chPos=list(((pos,char) for pos,val in enumerate(string) if(val == char)))
  except:
    pass
  return chPos
#提取关键词
def keyword_extract(data,filename):
    tfidf=analyse.extract_tags
    keywords=tfidf(data)
    return keywords
#将文档的每句话进行关键词提取，并将结果保存在txt文件中
def getKeywords(docpath,savepath):
    with open(docpath,'r',encoding='utf-8') as docf,open(savepath,'w',encoding='utf-8') as outf:
        for data in docf:#每句话
            data=data[:len(data)-1]
            keywords=keyword_extract(data,savepath)
            for word in keywords:
                outf.write(word+'/')
            outf.write('\n ')
#利用训练好的词向量获取关键词的词向量
import codecs
def word2vec(file_name,model):


    DataFile = codecs.open(file_name, "r",encoding='utf-8')
    DataSet = DataFile.readlines()[:-1]

    wordvec_size =100
    word_vec_all = numpy.zeros(wordvec_size)


    for data in DataSet:#

            space_pos=get_char_pos(data,'/')
            first_word=data[0:space_pos[0][0]]
            if first_word in model:


                word_vec_all=word_vec_all+model[first_word]


            for i in range(len(space_pos)-2):
                word=data[space_pos[i][0]:space_pos[i+1][0]]

                try:
                    c=model[word]

                except KeyError:
                    c=0
                word_vec_all=word_vec_all+c


    return word_vec_all
#词向量相似度计算代码：余弦
def simlarityCalu(vector1,vector2):
    vector1Mod=numpy.sqrt(vector1.dot(vector1))
    vector2Mod=numpy.sqrt(vector2.dot(vector2))
    if vector2Mod!=0 and vector1Mod!=0:
        simlarity=(vector1.dot(vector2))/(vector1Mod*vector2Mod)
    else:
        simlarity=0
    return simlarity


if __name__=='__main__':
    #下载模型
    model=gensim.models.Word2Vec.load('zhiwiki_news.word2vec')
    p1='P1.txt'
    p2='P2.txt'
    p1_keywords='P1_keyword.txt'
    p2_keywords = 'P2_keyword.txt'
    #获取关键词
    getKeywords(p1,p1_keywords)
    getKeywords(p2,p2_keywords)
    p1_vec=word2vec(p1_keywords,model)
    p2_vec=word2vec(p2_keywords,model)
    #计算相似度
    print(simlarityCalu(p1_vec,p2_vec))

转载于:https://www.cnblogs.com/hapyygril/p/9981788.html