c++实现布隆过滤器

hashfuntion.h（常用字符串Hash函数）

unsigned int SDBMHash(const char *str);
unsigned int RSHash(const char *str);
unsigned int JSHash(const char *str);
unsigned int PJWHash(const char *str);
unsigned int APHash(const char *str);
unsigned int DJBHash(const char *str);
unsigned int ELFHash(const char *str);
unsigned int BKDRHash(const char *str);

hashfuntion.cpp

#include "hashfunction.h"

unsigned int SDBMHash(const char *str)
{
    unsigned int hash = 0;
    while (*str)
    {
        // equivalent to: hash = 65599*hash + (*str++);
        hash = (*str++) + (hash << 6) + (hash << 16) - hash;
    }
    return (hash & 0x7FFFFFFF);
}

//RS Hash Function
unsigned int RSHash(const char *str)
{
    unsigned int b = 378551;
    unsigned int a = 63689;
    unsigned int hash = 0;
    while (*str)
    {
        hash = hash * a + (*str++);
        a *= b;
    }
    return (hash & 0x7FFFFFFF);
}

//JS Hash Function
unsigned int JSHash(const char *str)
{
    unsigned int hash = 1315423911;
    while (*str)
    {
        hash ^= ((hash << 5) + (*str++) + (hash >> 2));
    }
    return (hash & 0x7FFFFFFF);
}

//P. J. Weinberger Hash Function
unsigned int PJWHash(const char *str)
{
    unsigned int BitsInUnignedInt = (unsigned int)(sizeof(unsigned int) * 8);
    unsigned int ThreeQuarters    = (unsigned int)((BitsInUnignedInt  * 3) / 4);
    unsigned int OneEighth        = (unsigned int)(BitsInUnignedInt / 8);
    unsigned int HighBits         = (unsigned int)(0xFFFFFFFF) << (BitsInUnignedInt - OneEighth);
    unsigned int hash             = 0;
    unsigned int test             = 0;
    while (*str)
    {
        hash = (hash << OneEighth) + (*str++);
        if ((test = hash & HighBits) != 0)
        {
            hash = ((hash ^ (test >> ThreeQuarters)) & (~HighBits));
        }
    }
    return (hash & 0x7FFFFFFF);
}

//AP Hash Function
unsigned int APHash(const char *str)
{
    unsigned int hash = 0;
    int i;
    for (i=0; *str; i++)
    {
        if ((i & 1) == 0)
        {
            hash ^= ((hash << 7) ^ (*str++) ^ (hash >> 3));
        }
        else
        {
            hash ^= (~((hash << 11) ^ (*str++) ^ (hash >> 5)));
        }
    }
    return (hash & 0x7FFFFFFF);
}

//DJB Hash Function
unsigned int DJBHash(const char *str)
{
    unsigned int hash = 5381;
    while (*str)
    {
        hash += (hash << 5) + (*str++);
    }
    return (hash & 0x7FFFFFFF);
}

// ELF Hash Function
unsigned int ELFHash(const char *str)
{
    unsigned int hash = 0;
    unsigned int x    = 0;
    while (*str)
    {
        hash = (hash << 4) + (*str++);
        if ((x = hash & 0xF0000000L) != 0)
        {
            hash ^= (x >> 24);
            hash &= ~x;
        }
    }
    return (hash & 0x7FFFFFFF);
}

//BKDR Hash Function
unsigned int BKDRHash(const char *str)
{
    unsigned int seed = 131; // 31 131 1313 13131 131313 etc..
    unsigned int hash = 0;
    while (*str)
    {
        hash = hash * seed + (*str++);
    }
    return (hash & 0x7FFFFFFF);
}

bloomfilter.h

#ifndef BLOOMFILTER_H
#define BLOOMFILTER_H

#include <stdio.h>
#include <iostream>
#include <string>
#include <math.h>
#include <vector>
#include <fstream>
#include "hashfunction.h"

#define INT_SIZE 32
#define BUFFER_SIZE 1024

class BloomFilter{
public:
    //样本个数n，期望的失误率p，传入样本的文档路径
    BloomFilter(double errRate,int sampleNum);
    ~BloomFilter();
    void filterInit();                      //初始化布隆过滤器
    void arrayGenerate(std::string path);   //打开path路径的文档，计算每一行样本到array中
    bool isContain(const char* str);        //查看字符串是否在样本中存在
    void storeArray(std::string path);      //把array存储在指定路径文件path里
    void restoreArray(std::string path);    //把array从指定路径文件path里恢复载入
    int getHashFunNum();                    //返回需要的哈希函数的个数k
    int getIntNum();                        //返回需要的内存的int数
private:
    int hashtableInit();    //把几个哈希函数加入到hastable中
    std::string path;       //传入样本的文档路径
    double errRate;         //样本失误率p
    int sampleNum;          //样本个数n
    int bitNum;             //需要的二进制位数m
    int intNum;             //需要申请内存的int数
    int hashFunNum;         //需要的哈希函数的个数k,注意计算得到的哈希函数个数k应该<=hashtable.size();
    int *array;             //内存
    std::vector<unsigned int (*)(const char*)> hashtable;    //存放计算字符串哈希值的哈希函数
};

#endif // BLOOMFILTER_H

bloomfilter.cpp

#include "bloomfilter.h"

BloomFilter::BloomFilter(double errRate,int sampleNum)
{
    this->errRate=errRate;
    this->sampleNum=sampleNum;
    this->bitNum=-((this->sampleNum*log(this->errRate))/(log(2)*log(2)));
    this->hashFunNum=0.7*(this->bitNum/this->sampleNum);
    this->intNum=this->bitNum/INT_SIZE+1;
    array=new int[this->intNum];
}

BloomFilter::~BloomFilter()
{
    delete []array;
}

void BloomFilter::filterInit()
{
    hashtableInit();
    if(this->hashFunNum>(int)hashtable.size())
    {
        std::cout<<"哈希函数不足,请添加"<<std::endl;
        return;
    }
}

int BloomFilter::hashtableInit()
{
    hashtable.push_back(*PJWHash);
    hashtable.push_back(*JSHash);
    hashtable.push_back(*RSHash);
    hashtable.push_back(*SDBMHash);
    hashtable.push_back(*APHash);
    hashtable.push_back(*DJBHash);
    hashtable.push_back(*BKDRHash);
    hashtable.push_back(*ELFHash);
    return hashtable.size();
}

void BloomFilter::arrayGenerate(std::string path)
{
    int hashval;
    std::fstream fs;
    fs.open(path.c_str(),std::ios::in|std::ios::out);
    if(!fs)
    {
        perror("open file error");
        return;
    }
    fs.seekp(std::ios::beg);
    char buf[BUFFER_SIZE]={0};
    fs.getline(buf,BUFFER_SIZE);
    while(!fs.eof())
    {
        for(int i=0;i!=this->hashFunNum;i++)
        {
            hashval=hashtable[i](buf);
            hashval=hashval%(this->intNum*INT_SIZE);
            this->array[hashval/INT_SIZE]|=(0x1<<(hashval%INT_SIZE));
        }
        fs.getline(buf,BUFFER_SIZE);
    }
    fs.clear();
    fs.close();
}

bool BloomFilter::isContain(const char* str)
{
    int hashval;
    for(int i=0;i!=this->hashFunNum;i++)
    {
        hashval=hashtable[i](str);
        hashval=hashval%(this->intNum*INT_SIZE);
        if(array[hashval/INT_SIZE]&(0x1<<(hashval%INT_SIZE)))
            continue;
        else
            return false;
    }
    return true;
}

void BloomFilter::storeArray(std::string path)
{
    std::fstream fs;
    fs.open(path.c_str(),std::ios::in|std::ios::out);
    if(!fs)
    {
        fs.open(path.c_str(),std::ios::app|std::ios::in|std::ios::out);
        for(int k=0;k<this->intNum;k++)
            fs<<array[k]<<std::endl;
    }
    else
    {
        fs.seekp(std::ios::beg);
        for(int k=0;k<this->intNum;k++)
            fs<<array[k]<<std::endl;
    }
    fs.clear();
    fs.close();
}

void BloomFilter::restoreArray(std::string path)
{
    std::fstream fs;
    fs.open(path.c_str(),std::ios::in|std::ios::out);
    if(!fs)
    {
        perror("open file error");
        return;
    }
    fs.seekp(std::ios::beg);
    char buf[INT_SIZE]={0};
    fs.getline(buf,INT_SIZE);
    int i=0;
    while(!fs.eof())
    {
        sscanf(buf,"%d\n",&this->array[i++]);
        fs.getline(buf,INT_SIZE);
    }
    fs.clear();
    fs.close();
}

int BloomFilter::getHashFunNum()
{
    return this->hashFunNum;
}

int BloomFilter::getIntNum()
{
    return this->intNum;
}

测试：

main.cpp

加载文件生成array并保存：

#include "tool/bloomfilter.h"
using namespace std;

int main(int argc, char *argv[])
{
    BloomFilter mybloom(0.01,100);
    mybloom.filterInit();
    mybloom.arrayGenerate("../tool/redlist.txt");
    cout<<"intNum:"<<mybloom.getIntNum()<<",hashFunNum:"<<mybloom.getHashFunNum()<<endl;
    cout<<"www.dubai.com在我的集合中吗:"<<(mybloom.isContain("www.dubai.com")?"在":"不在")<<endl;
    cout<<"www.qq.com在我的集合中吗:"<<(mybloom.isContain("www.qq.com")?"在":"不在")<<endl;
    mybloom.storeArray("../tool/bloom_array.txt");
    return 0;
}

从文件中读取恢复array：

#include "tool/bloomfilter.h"
using namespace std;

int main(int argc, char *argv[])
{
    BloomFilter mybloom(0.01,100);
    mybloom.filterInit();
    mybloom.restoreArray("../tool/bloom_array.txt");
    cout<<"www.dubai.com在我的集合中吗:"<<(mybloom.isContain("www.dubai.com")?"在":"不在")<<endl;
    cout<<"www.qq.com在我的集合中吗:"<<(mybloom.isContain("www.qq.com")?"在":"不在")<<endl;
    return 0;
}