基于字的文本相似度算法——Jacard算法

一、算法原理

基于字的文本相似度Jacard 算法的原理是：
（1）计算两个文本中字的交集
（2）计算两个文本中字的并集
（3）交集内的字的个数除以并集内的字的个数即为文本相似度值
（4)根据设置的阈值判断是否相似

二、算法的C++实现

这里引用的StringUtil.hpp文件引自：

https://github.com/yanyiwu/cppjieba/blob/master/deps/limonp/StringUtil.hpp

/*
 * JaccardSimilarity.hpp
 *
 *  Created: 2016年10月2日
 *   Author: tang
 */

#ifndef SRC_JACCARD_SIMILARITY_HPP_
#define SRC_JACCARD_SIMILARITY_HPP_
#include <algorithm>
#include <iostream>
#include <vector>
#include <set>
#include "StringUtil.hpp"

using namespace std;

class JaccardSimilarity
{
public:
	
	JaccardSimilarity()
	{
	}

	double CalculateTextSimilarity(string &str1,string &str2)
	{
		vector<uint16_t> words_for_str1;
		vector<uint16_t> words_for_str2;
		vector<uint16_t>::iterator it;

		if(!utf8ToUnicode< vector<uint16_t> >(str1,words_for_str1) || 
			!utf8ToUnicode< vector<uint16_t> >(str2,words_for_str2 ) )
		{
			cout<<"TransCode Error"<<endl;
			return 0.;
		}

		for(it=words_for_str1.begin();it!=words_for_str1.end();)
		{
			if(codeFilter(*it))
			{
				++it;
			}
			else
			{
				it=words_for_str1.erase(it);
			}
		}

		for(it=words_for_str2.begin();it!=words_for_str2.end();)
                {
                        if(codeFilter(*it))
                        {
				++it;
                        }
			else
			{
				it=words_for_str2.erase(it);
			}
                }

		if(words_for_str1.size()+words_for_str2.size()<1)
			return 1.;

		vector<uint16_t> words_intersection;
		vector<uint16_t> words_union;
		std::sort(words_for_str1.begin(),words_for_str1.end());
		std::sort(words_for_str2.begin(),words_for_str2.end());
		std::set_intersection(words_for_str1.begin(),words_for_str1.end(),
					words_for_str2.begin(),words_for_str2.end(),
					std::inserter(words_intersection,words_intersection.begin()));

		std::set_union(words_for_str1.begin(),words_for_str1.end(),
                                        words_for_str2.begin(),words_for_str2.end(),
					std::inserter(words_union,words_union.begin()));

		double inter=words_intersection.size();
		double wunion=words_union.size();

		return inter/wunion;
	}

	bool codeFilter(int code) 
	{
        	if ((code < 0x4e00 || code > 0x9fa5) && 
			!(code >= '0' && code <= '9') && 
			!(code >= 'a' && code <= 'z') && 
			!(code >= 'A' && code <= 'Z'))
           		 return false;
        
        	return true;
	}

};

#endif /* SRC_JACCARD_SIMILARITY_HPP_ */

三、算法的java实现

import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;


public class JaccardSimilarity{

    public JaccardSimilarity() {
    }
    
    public boolean codeFilter(int code) {
        if ((code < 19968 || code > 40869) 
		&& !(code >= '0' && code <= '9') 
		&& !(code >= 'a' && code <= 'z') 
		&& !(code >= 'A' && code <= 'Z')) {
            return false;
        }
        return true;
    }

    public double CalculateTextSim(String content, String compareContent) {
        if(null == content || null == compareContent)
            return 0.0;
        Map<String, Integer> cntMap = new HashMap<String, Integer>();
        Set<String> cntSet = new HashSet<String>();
        Map<String, Integer> cmpCntMap = new HashMap<String, Integer>();
        Set<String> cmpCntSet = new HashSet<String>();
        
	for (int i = 0; i != content.length(); i++) {
            int k = 0;
            if (codeFilter(content.codePointAt(i))) {
                if (cntMap.containsKey("" + content.charAt(i))) {
                    Integer count = cntMap.get("" + content.charAt(i));
                    count = count + 1;
                    cntMap.put("" + content.charAt(i), count);
                    k = count;
                } else {
                    cntMap.put("" + content.charAt(i), new Integer(1));
                    k = 1;
                }
                String tmpString = content.charAt(i) + "" + k;
                cntSet.add(tmpString);
            }
        }

        for (int i = 0; i != compareContent.length(); i++) {
            int k = 0;
            if (codeFilter(compareContent.codePointAt(i))) {
                if (cmpCntMap.containsKey("" + compareContent.charAt(i))) {
                    Integer count = cmpCntMap.get("" + compareContent.charAt(i));
                    count = count + 1;
                    cmpCntMap.put("" + compareContent.charAt(i), count);
                    k = count;
                } else {
                    cmpCntMap.put("" + compareContent.charAt(i), new Integer(1));
                    k = 1;
                }

                String tmpString = compareContent.charAt(i) + "" + k;
                cmpCntSet.add(tmpString);
            }
        }

        Set<String> tmpSet = new HashSet<String>();
        tmpSet.addAll(cntSet);
        cntSet.retainAll(cmpCntSet);
        double intCount = cntSet.size();

        tmpSet.addAll(cmpCntSet);


        if (tmpSet.size() == 0)
            return 0;
        double uniCount = tmpSet.size();


        return intCount / uniCount;
    }

}