余弦相似度

import scipy
import torch

def calculate_distances(features1,features2):
    return scipy.spatial.distance.cdist(features1,features2, "cosine")


def cosine_similarity_torch(features1, features2):
    # 转为tensor
    features1 = torch.tensor(features1)
    features2 = torch.tensor(features2)
    # 从cpu – > gpu
    features1 = features1.cuda()
    features2 = features2.cuda()
    # 求二范数
    features1_norm = torch.norm(features1, 2, 1, True)
    features2_norm = torch.norm(features2, 2, 1, True)
    # 求scipy.spatial.distance.cosine的公式表达
    cos = 1 - torch.mm(features1, features2.t()) / (torch.mm(features1_norm, features2_norm.t()))
    # 将tensor转为cpu的numpy
    distances = cos.cpu().numpy()
    return distances

import jieba
import time
from scipy import spatial
import numpy as np

from sentence_similarity.Utils.load_data import *

file_voc = './sentence_similarity/data/voc.txt'
file_idf = './sentence_similarity/data/idf.txt'
file_userdict = './sentence_similarity/data/medfw.txt'


class SSIM(object):

    def __init__(self):
        t1 = time.time()
        self.voc=load_voc(file_voc)
        print("Loading  word2vec vector cost %.3f seconds...\n" % (time.time() - t1))
        t1 = time.time()
        self.idf = load_idf(file_idf)
        print("Loading  idf data cost %.3f seconds...\n" % (time.time() - t1))
        jieba.load_userdict(file_userdict)

    def M_cosine(self,s1,s2):
        s1_list = jieba.lcut(s1)
        s2_list = jieba.lcut(s2)

        v1 = np.array([self.voc[s] for s in s1_list if s in self.voc])
        v2 = np.array([self.voc[s] for s in s2_list if s in self.voc])

        v1 = v1.sum(axis=0)
        v2 = v2.sum(axis=0)

        sim = 1-spatial.distance.cosine(v1,v2)
        return sim

    def M_idf(self,s1, s2):
        v1, v2 = [], []
        s1_list = jieba.lcut(s1)
        s2_list = jieba.lcut(s2)

        for s in s1_list:
            idf_v = self.idf.get(s, 1)
            if s in self.voc:
                v1.append(1.0 * idf_v * self.voc[s])

        for s in s2_list:
            idf_v = self.idf.get(s, 1)
            if s in self.voc:
                v2.append(1.0 * idf_v * self.voc[s])

        v1 = np.array(v1).sum(axis=0)
        v2 = np.array(v2).sum(axis=0)

        sim = 1 - spatial.distance.cosine(v1, v2)
        return sim

reference

https://docs.scipy.org/doc/scipy-0.17.1/reference/generated/scipy.spatial.distance.cosine.html#scipy.spatial.distance.cosine

https://github.com/cjymz886/sentence-similarity