1.4 布隆过滤器

布隆过滤器是一种概率型数据结构，能明确告知某个字符串一定不存在或者可能存在，直观说是hash set，不存储key，通过k个hash值，存储k个标志位，k个标志位均为1，表示可能存在；否则一定不存在。

 

布隆过滤器解决的问题是，海量数据查找。红黑树的需要key比较和存储key；哈希表的存储key，以及碰撞后的key比较。所以需要尝试探寻不存储key的方案，并且拥有哈希表的优点（不需要比较字符串）。

 

布隆过滤器优点：需要存储空间小；不需要比较，查询快。

布隆过滤器缺点：不支持删除操作；key有概率不存在集合里。

 

应用场景：

 

 

几个概念：

1、负载因子：数组存储元素的个数/数组长度；负载因子越小，冲突越小；负载因子越大，冲突越大。既是占用率。

 

2、哈希函数：把任意长度的而输入通过散列算法换成固定长度的输出。要求：计算速度快；相似字符串能保持强随机分布型（防碰撞）

例如，MD5、SHA1、SHA256，一般用于信息安全（签名和加密）；MurMurHash，CityHash，不适用于加密，但适合用在散列表

 

3、线性探查与同类哈希聚集：不同key，映射到哈希表的相同位置时，线性探查采用i+1，i+2的方式找空位，会导致同类哈希的聚集。

4、二次探查：采用方式为i+1²，i-1²，i+2²，i-2²，i+3²，i-3²......延缓同类哈希聚集，仍会导致同类哈希聚集。

5、二度哈希（双重哈希）：

在.net HashTable类的hash函数Hk定义如下：

Hk(key) = [GetHash(key) + k * (1 + (((GetHash(key) >> 5) + 1) % (hashsize – 1)))] % hashsize

https://www.cnblogs.com/organic/p/6283476.html

 

6、n，p，m，k推导过程

 

代码：

#include <stdio.h>
#include <stdint.h>
#include <math.h>
#include <stdlib.h>
#include <string.h>

#define BYTE_BITS (8)
#define MIX_UINT64(v)       ((uint32_t)((v>>32)^(v)))

#define SETBIT(filter, n)   (filter->pstFilter[n/BYTE_BITS] |= (1 << (n%BYTE_BITS)))
#define GETBIT(filter, n)   (filter->pstFilter[n/BYTE_BITS] & (1 << (n%BYTE_BITS)))

typedef struct {
    uint32_t dwMaxItems;                            // n - BloomFilter中最大元素个数 (输入量)
    double dProbFalse;                              // p - 假阳概率 (输入量，比如万分之一：0.00001)
    uint32_t dwFilterBits;                          // m = ceil((n * log(p)) / log(1.0 / (pow(2.0, log(2.0))))); - BloomFilter的比特数
    uint32_t dwHashFuncs;                           // k = round(log(2.0) * m / n); - 哈希函数个数

    uint32_t dwSeed;                                // MurmurHash的种子偏移量

    uint32_t dwFilterSize;                          // dwFilterBits / BYTE_BITS
    unsigned char *pstFilter;                       // BloomFilter存储指针，使用malloc分配

    uint32_t *pdwHashPos;                           // 存储上次hash得到的K个bit位置数组(由bloom_hash填充)
}BaseBloomFilter;



void _CalcBloomFilterParam(uint32_t n,double p, uint32_t *pm,uint32_t *pk){
    /*
    n -- 布隆过滤器中元素的个数，如上图 只有str1和str2 两个元素 那么 n=2
    p -- 假阳率，在0-1之间 0.000000
    m -- 位图所占空间
    k -- hash函数的个数
    公式如下：
    n = ceil(m / (-k / log(1 - exp(log(p) / k))))
    p = pow(1 - exp(-k / (m / n)), k)
    m = ceil((n * log(p)) / log(1 / pow(2, log(2))));
    k = round((m / n) * log(2));
    */
    uint32_t m, k;

    m = (uint32_t) ceil(-1.0 * n * log(p) / 0.480453 );//log以e为底的对数函数
    m = (m - m%64 ) + 64;//8字节对齐

    k = (uint32_t) round(0.69314 * m / n);

    *pm = m;
    *pk = k;
}

int InitBloomFilter(BaseBloomFilter* pstBloomfilter, \
    uint32_t dwSeed, uint32_t dwMaxItems, double dProbFalse){
    
    if(pstBloomfilter == NULL)
        return -1;
    
    if(dProbFalse <=0 || dProbFalse >= 1){
        return -2;
    }

    if(pstBloomfilter->pstFilter != NULL){
        free(pstBloomfilter->pstFilter);
    }
    if(pstBloomfilter->pdwHashPos != NULL){
        free(pstBloomfilter->pdwHashPos);
    }

    pstBloomfilter->dwMaxItems = dwMaxItems;
    pstBloomfilter->dProbFalse = dProbFalse;
    pstBloomfilter->dwSeed = dwSeed;

    _CalcBloomFilterParam(pstBloomfilter->dwMaxItems,pstBloomfilter->dProbFalse,\
                           &pstBloomfilter->dwFilterBits,&pstBloomfilter->dwHashFuncs);
    

    pstBloomfilter->dwFilterSize = pstBloomfilter->dwFilterBits / BYTE_BITS;
    pstBloomfilter->pstFilter = (unsigned char *) malloc(pstBloomfilter->dwFilterSize);
    if(NULL == pstBloomfilter->pstFilter){
        return -100;
    }

    pstBloomfilter->pdwHashPos = (uint32_t *) malloc(sizeof(uint32_t) * pstBloomfilter->dwHashFuncs);
    if(NULL == pstBloomfilter->pdwHashPos){
        return -200;
    }

    printf(">>> Init BloomFilter(n=%u, p=%e, m=%u, k=%d), malloc() size=%.2fMB, items:bits=1:%0.1lf\n",
           pstBloomfilter->dwMaxItems, pstBloomfilter->dProbFalse, pstBloomfilter->dwFilterBits,
           pstBloomfilter->dwHashFuncs, (double)pstBloomfilter->dwFilterSize/1024/1024,
           pstBloomfilter->dwFilterBits*1.0/pstBloomfilter->dwMaxItems);

    memset(pstBloomfilter->pstFilter,0,pstBloomfilter->dwFilterSize);
    return 0;
}

int FreeBloomFilter(BaseBloomFilter *pstBloomfilter){
    if(pstBloomfilter == NULL){
        return -1;
    }

    free(pstBloomfilter->pstFilter);
    pstBloomfilter->pstFilter = NULL;
    free(pstBloomfilter->pdwHashPos);
    pstBloomfilter->pdwHashPos = NULL;
    return 0;
}


// MurmurHash2, 64-bit versions, by Austin Appleby
// https://sites.google.com/site/murmurhash/
uint64_t MurmurHash2_x64 ( const void * key, int len, uint32_t seed )
{
    const uint64_t m = 0xc6a4a7935bd1e995;
    const int r = 47;

    uint64_t h = seed ^ (len * m);

    const uint64_t * data = (const uint64_t *)key;
    const uint64_t * end = data + (len/8);

    while(data != end)
    {
        uint64_t k = *data++;

        k *= m;
        k ^= k >> r;
        k *= m;

        h ^= k;
        h *= m;
    }

    const uint8_t * data2 = (const uint8_t*)data;

    switch(len & 7)
    {
    case 7: h ^= ((uint64_t)data2[6]) << 48;
    case 6: h ^= ((uint64_t)data2[5]) << 40;
    case 5: h ^= ((uint64_t)data2[4]) << 32;
    case 4: h ^= ((uint64_t)data2[3]) << 24;
    case 3: h ^= ((uint64_t)data2[2]) << 16;
    case 2: h ^= ((uint64_t)data2[1]) << 8;
    case 1: h ^= ((uint64_t)data2[0]);
        h *= m;
    };

    h ^= h >> r;
    h *= m;
    h ^= h >> r;

    return h;
}

// 双重散列封装
void bloom_hash(BaseBloomFilter *pstBloomfilter, const void * key, int len)
{
    //if (pstBloomfilter == NULL) return;
    int i;
    uint32_t dwFilterBits = pstBloomfilter->dwFilterBits;
    uint64_t hash1 = MurmurHash2_x64(key, len, pstBloomfilter->dwSeed);
    uint64_t hash2 = MurmurHash2_x64(key, len, MIX_UINT64(hash1));

    for (i = 0; i < (int)pstBloomfilter->dwHashFuncs; i++)
    {
        pstBloomfilter->pdwHashPos[i] = (hash1 + i*hash2) % dwFilterBits;
    }

    return;
}

int BloomFilter_Add(BaseBloomFilter *pstBloomfilter, const void* key,int len){
    if(pstBloomfilter == NULL || key == NULL || len<=0){
        return -1;
    }

    bloom_hash(pstBloomfilter,key,len);
    int i;
    for(i=0;i<(int)pstBloomfilter->dwHashFuncs;i++){
        SETBIT(pstBloomfilter,pstBloomfilter->pdwHashPos[i]);
    }

    return 0;
}

int BloomFilter_Check(BaseBloomFilter *pstBloomfilter, const void* key, int len){
    if(pstBloomfilter == NULL || key == NULL || len<=0){
        return -1;
    }

    bloom_hash(pstBloomfilter,key,len);
    int i;
    for(i=0;i<(int)pstBloomfilter->dwHashFuncs;i++){
        if(GETBIT(pstBloomfilter,pstBloomfilter->pdwHashPos[i]) == 0){
            return 1;//没找到
        }
    }

    return 0;
}

int main(){
    int MAX_ITEMS = 400000;
    int ADD_ITEMS = 1000;
    double P_ERROR = 0.0000001;
    BaseBloomFilter stBloomfilter = {0};
    InitBloomFilter(&stBloomfilter,0,MAX_ITEMS,P_ERROR);

    char url[128] = {0};
    int i;
    for(i=0;i<ADD_ITEMS;i++){
        sprintf(url,"https://0voice.com/%d.html", i);
        BloomFilter_Add(&stBloomfilter,url,strlen(url));
    }

    printf("0 exist, 1 not exist\n");
    sprintf(url,"https://0voice.com/%d.html",0);
    printf("%s exist %d\n",url,BloomFilter_Check(&stBloomfilter,url,strlen(url)));

    sprintf(url,"https://0voice.com/%d.html",ADD_ITEMS);
    printf("%s exist %d\n",url,BloomFilter_Check(&stBloomfilter,url,strlen(url)));
}

参考

零声学院

https://www.cnblogs.com/liyulong1982/p/6013002.html

https://www.cnblogs.com/wt869054461/p/5731577.html

 

MurMurhash

https://sites.google.com/site/murmurhash/

https://github.com/aappleby/smhasher

CityHash

https://github.com/google/cityhash

 

双重哈希

https://www.cnblogs.com/organic/p/6283476.html