DirectDiskUrlFilter

---

package com.xiongbeer.webveins.filter;

import com.google.common.hash.Funnel;
import com.google.common.hash.Hashing;
import com.google.common.primitives.Longs;
import com.google.common.primitives.UnsignedBytes;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;

/**
* 牺牲吞吐效率（性能瓶颈在IO）来换取强一致性和无大小限制的BloomFilter，直接在硬盘上随机读写，不载入内存
*
* Guava版本 21.0
*
*
* Guava中BloomFilter序列化格式：
* +———————————————————————–+
* | strategyOrdinal | numHashFunctions | dataLength | BitArray |
* | 1 byte | 1 byte | 1 int | 8*dataLength bytes |
* +———————————————————————–+
* 采用的numHashFunctions个Hash函数实现：
* 首先使用murmurHash3对Object生成一个128位的值，取低64位值为combinedHash
* 1.mask combinedHash的符号位后对BitArray.Size进行取余后的值就是新的置true的位置
* 2.combinedHash更新为combinedHash += 高64位的值
* 按上2步骤循环numHashFunctions次
*
* Created by shaoxiong on 17-7-1.
*/
public class DirectDiskUrlFilter {
private Funnel funnel;
private File filterFile;
private RandomAccessFile raf;
private int numHashFunctions;
private int dataLength;
private long bitsSize;
private Bits bits;

/* 1 byte + 1 byte + 1 int */
static final int OPERATION_OFFSET = 6;

/**
 *
 * @param filterPath 原Guava序列化存储的文件路径
 * @param funnel     原Guava BloomFilter使用的Funnel
 * @throws IOException
 */
public DirectDiskUrlFilter(String filterPath, Funnel<CharSequence> funnel) throws IOException {
    filterFile = new File(filterPath);
    raf = new RandomAccessFile(filterFile, "rw");
    /* jump strategyOrdinal value */
    raf.readByte();
    numHashFunctions = UnsignedBytes.toInt(raf.readByte());
    dataLength = raf.readInt();
    bitsSize = (long)dataLength * 64L;
    bits = new Bits();
    this.funnel = funnel;
}

public boolean put(String url) throws IOException {
    byte[] bytes = Hashing.murmur3_128().hashObject(url, funnel).asBytes();
    long hash1 = this.lowerEight(bytes);
    long hash2 = this.upperEight(bytes);
    boolean bitsChanged = false;
    long combinedHash = hash1;

    for(int i = 0; i < numHashFunctions; ++i) {
        /* mask combinedHash with 0x7FFFFFFFFFFFFFFF */
        bitsChanged |= bits.set((combinedHash & 9223372036854775807L) % bitsSize);
        combinedHash += hash2;
    }

    return bitsChanged;
}

public boolean mightContain(String url) throws IOException {
    byte[] bytes = Hashing.murmur3_128().hashObject(url, funnel).asBytes();
    long hash1 = this.lowerEight(bytes);
    long hash2 = this.upperEight(bytes);
    long combinedHash = hash1;

    for(int i = 0; i < numHashFunctions; ++i) {
        if(!bits.get((combinedHash & 9223372036854775807L) % bitsSize)) {
            return false;
        }

        combinedHash += hash2;
    }

    return true;
}

/**
 * 使用完毕要记得关闭IO流
 *
 * @throws IOException
 */
public void close() throws IOException {
    raf.close();
}

private long lowerEight(byte[] bytes) {
    return Longs.fromBytes(bytes[7], bytes[6], bytes[5], bytes[4], bytes[3], bytes[2], bytes[1], bytes[0]);
}

private long upperEight(byte[] bytes) {
    return Longs.fromBytes(bytes[15], bytes[14], bytes[13], bytes[12], bytes[11], bytes[10], bytes[9], bytes[8]);
}

/**
 * ((index >>> 6) << 3) 与 index >>> 3 的效果完全是不一样的，请特别注意
 * 由于存储的时候 1 byte = 8 bits，所以这里重写了定位（取余）的算法
 */
final class Bits {

    boolean set(long index) throws IOException {
        long blockValue = this.getBlockValue(index);
        if(blockValue != -1) {
            raf.seek(OPERATION_OFFSET + ((index >>> 6) << 3));
            raf.writeLong(blockValue | (1L << (int)index));
            return true;
        } else {
            return false;
        }
    }

    boolean get(long index) throws IOException {
        raf.seek(OPERATION_OFFSET + ((index >>> 6) << 3));
        return (raf.readLong() & (1L << (int)index)) != 0L;
    }

    long getBlockValue(long index) throws IOException {
        raf.seek(OPERATION_OFFSET + ((index >>> 6) << 3));
        long blockValue = raf.readLong();
        if((blockValue & (1L << (int)index)) == 0L){
            return blockValue;
        }
        return -1;
    }
}

}

---