java利用Simhash做分词对比文本处理

 

SimHash是一种用于比较文本相似度的算法，其核心思想是将文本转换为一个定长的二进制向量，然后通过计算向量之间的汉明距离来衡量文本之间的相似度。

Java中实现SimHash算法可以使用如下步骤：

1. 分词：使用分词器将文本分成单词或者词组。

2. 提取关键词：对于每个单词或者词组，使用哈希函数计算其哈希值，并且对每一位二进制位进行加权求和，得到一个定长的二进制向量。

3. 计算SimHash：将所有关键词的二进制向量按位相加，并且对每一位进行比较，如果大于等于0，则该位为1，否则为0，得到一个定长的SimHash值。

import java.math.BigInteger;

public class SimHash {
    private String text;
    private BigInteger hash;
    private int hashbits;

    public SimHash(String text, int hashbits) {
        this.text = text;
        this.hashbits = hashbits;
        this.hash = this.simHash();
    }

    public BigInteger simHash() {
        int[] hashbits = new int[this.hashbits];
        String[] words = this.text.split("\\s+");
        for (String word : words) {
            BigInteger wordHash = this.hash(word);
            for (int i = 0; i < this.hashbits; i++) {
                BigInteger bitmask = BigInteger.valueOf(1).shiftLeft(i);
                if (wordHash.and(bitmask).signum() != 0) {
                    hashbits[i] += 1;
                } else {
                    hashbits[i] -= 1;
                }
            }
        }
        BigInteger fingerprint = BigInteger.ZERO;
        for (int i = 0; i < this.hashbits; i++) {
            if (hashbits[i] >= 0) {
                fingerprint = fingerprint.add(BigInteger.valueOf(1).shiftLeft(i));
            }
        }
        return fingerprint;
    }

    private BigInteger hash(String word) {
        if (word == null || word.length() == 0) {
            return BigInteger.ZERO;
        }
        char[] wordArray = word.toCharArray();
        BigInteger x = BigInteger.valueOf(((long) wordArray[0]) << 7);
        BigInteger m = BigInteger.valueOf(1000003);
        BigInteger mask = BigInteger.valueOf(2).pow(this.hashbits).subtract(BigInteger.ONE);
        for (char c : wordArray) {
            BigInteger temp = BigInteger.valueOf((long) c);
            x = x.multiply(m).xor(temp).and(mask);
        }
        x = x.xor(BigInteger.valueOf(word.length()));
        if (x.equals(BigInteger.valueOf(-1))) {
            x = BigInteger.valueOf(-2);
        }
        return x;
    }

    public int hammingDistance(SimHash other) {
        BigInteger x = this.hash.xor(other.hash);
        int distance = 0;
        while (x.signum() != 0) {
            distance += 1;
            x = x.and(x.subtract(BigInteger.ONE));
        }
        return distance;
    }

    public static void main(String[] args) {
        String text1 = "中标公司";
        String text2 = "招标公司";
        SimHash simHash1 = new SimHash(text1, 64);
        SimHash simHash2 = new SimHash(text2, 64);
        int distance = simHash1.hammingDistance(simHash2);
        System.out.println("Hamming distance: " + distance);
    }
}