Redis布隆过滤器

Redis布隆过滤器

1.实现原理

• 布隆过滤器解析出设置值的offset数组
• Redis使用bitmap存储offsert

2.布隆过滤器初始化配置

@Configuration
@AutoConfigureAfter(RedisAutoConfiguration.class)
@SuppressWarnings("ALL")
public class RedisConfiguration {
 
    @Value("${bloom.filter.totalCount:10000000}")
    private long bloomTotalCount;
    @Value("${bloom.filter.error:0.01}")
    private double error;
 
    @Bean
    public RedisTemplate<String, Object> redisTemplate(LettuceConnectionFactory redisConnectionFactory) {
        Jackson2JsonRedisSerializer<Object> serializer = new Jackson2JsonRedisSerializer<Object>(Object.class);
        ObjectMapper objectMapper = new ObjectMapper();
        objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
        objectMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL);
        serializer.setObjectMapper(objectMapper);
 
        RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
        redisTemplate.setConnectionFactory(redisConnectionFactory);
        redisTemplate.setKeySerializer(new StringRedisSerializer());
        redisTemplate.setValueSerializer(serializer);
        redisTemplate.setHashKeySerializer(new StringRedisSerializer());
        redisTemplate.setHashValueSerializer(serializer);
        redisTemplate.afterPropertiesSet();
 
        return redisTemplate;
    }
 
    //初始化布隆过滤器，放入到spring容器里面
    @Bean
    public BloomFilterHelper<String> initBloomFilterHelper() {
        return new BloomFilterHelper<>((Funnel<String>) (from, into) -> into.putString(from, Charsets.UTF_8).putString(from, Charsets.UTF_8), bloomTotalCount, error);
    }
}

3. 布隆过滤器工具类（BloomFilterHelper）

package com.irobotics.aiot.util;
 
import com.google.common.base.Preconditions;
import com.google.common.hash.Funnel;
import com.google.common.hash.Hashing;
/**
 * @author guoliang.wu
 * @date 2022/6/20 18:24
 * @desc BloomFilterHelper
 */
public class BloomFilterHelper<T> {
    private int numHashFunctions;
 
    private int bitSize;
 
    private Funnel<T> funnel;
 
 
    public BloomFilterHelper(Funnel<T> funnel, long expectedInsertions, double fpp) {
        Preconditions.checkArgument(funnel != null, "funnel不能为空");
        this.funnel = funnel;
        // 计算bit数组长度
        bitSize = optimalNumOfBits(expectedInsertions, fpp);
        // 计算hash方法执行次数
        numHashFunctions = optimalNumOfHashFunctions(expectedInsertions, bitSize);
    }
 
 
    public int[] murmurHashOffset(T value) {
        int[] offset = new int[numHashFunctions];
        long hash64 = Hashing.murmur3_128().hashObject(value, funnel).asLong();
        int hash1 = (int) hash64;
        int hash2 = (int) (hash64 >>> 32);
        for (int i = 1; i <= numHashFunctions; i++) {
            int nextHash = hash1 + i * hash2;
            if (nextHash < 0) {
                nextHash = ~nextHash;
            }
            offset[i - 1] = nextHash % bitSize;
        }
        return offset;
    }
 
    /**
     * 计算bit数组长度
     */
    private int optimalNumOfBits(long n, double p) {
        if (p == 0) {
            // 设定最小期望长度
            p = Double.MIN_VALUE;
        }
        int sizeOfBitArray = (int) (-n * Math.log(p) / (Math.log(2) * Math.log(2)));
        return sizeOfBitArray;
    }
 
 
    /**
     * 计算hash方法执行次数
     */
    private int optimalNumOfHashFunctions(long n, long m) {
        int countOfHash = Math.max(1, (int) Math.round((double) m / n * Math.log(2)));
        return countOfHash;
    }
}

4. Redis布隆过滤器

package com.irobotics.aiot.util;
 
import com.google.common.base.Preconditions;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.ArrayUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.dao.DataAccessException;
import org.springframework.data.redis.core.RedisOperations;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.SessionCallback;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.stereotype.Service;
 
import java.util.Arrays;
import java.util.List;
 
/**
 * @author guoliang.wu
 * @date 2022/6/20 18:24
 * @desc RedisBloomFilter
 */
@Service
@Slf4j
public class RedisBloomFilter {
 
    @Autowired
    private RedisTemplate redisTemplate;
 
 
    /**
     * 根据给定的布隆过滤器添加值
     */
    public <T> void addByBloomFilter(BloomFilterHelper<T> bloomFilterHelper, String key, T value) {
        Preconditions.checkArgument(bloomFilterHelper != null, "bloomFilterHelper不能为空");
        int[] offset = bloomFilterHelper.murmurHashOffset(value);
        if (ArrayUtils.isNotEmpty(offset)) {
            redisTemplate.executePipelined(new SessionCallback<String>() {
                @Override
                public <K, V> String execute(RedisOperations<K, V> operations) throws DataAccessException {
                    ValueOperations<String, Object> valueOperations = (ValueOperations<String, Object>) operations.opsForValue();
                    Arrays.stream(offset).forEach(o -> {
                        valueOperations.setBit(key, o, true);
                    });
                    return null;
                }
            });
        }
    }
 
 
    /**
     * 根据给定的布隆过滤器判断值是否存在
     */
    public <T> boolean includeByBloomFilter(BloomFilterHelper<T> bloomFilterHelper, String key, T value) {
        Preconditions.checkArgument(bloomFilterHelper != null, "bloomFilterHelper不能为空");
        int[] offset = bloomFilterHelper.murmurHashOffset(value);
        List<Boolean> list = redisTemplate.executePipelined(new SessionCallback<Boolean>() {
            @Override
            public <K, V> Boolean execute(RedisOperations<K, V> operations) throws DataAccessException {
                ValueOperations<String, Object> valueOperations = (ValueOperations<String, Object>) operations.opsForValue();
                for (int i : offset) {
                    valueOperations.getBit(key, i);
                }
                return null;
            }
        });
        if (list.contains(Boolean.FALSE)) {
            return Boolean.FALSE;
        }
        return Boolean.TRUE;
    }
}

5.使用示范

package com.irobotics.aiot.controller;
 
import com.irobotics.aiot.util.BloomFilterHelper;
import com.irobotics.aiot.util.RedisBloomFilter;
import io.swagger.annotations.Api;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
 
import javax.servlet.http.HttpServletResponse;
 
/**
 * @author guoliang.wu
 * @date 2022/6/21 18:24
 * @desc 测试布隆过滤器
 */
@Api(tags = "测试布隆过滤器")
@RestController
@RequestMapping("/test")
@Slf4j
public class TestController {
 
    @Autowired
    private RedisBloomFilter redisBloomFilter;
 
    @Autowired
    private BloomFilterHelper<String> bloomFilterHelper;
 
 
    @ResponseBody
    @GetMapping("/checkOne")
    public boolean check(@RequestParam("num") String num, HttpServletResponse response) {
        boolean flag = redisBloomFilter.includeByBloomFilter(bloomFilterHelper, "bf:device_shadow", String.valueOf(num));
        if (!flag) {
            response.setStatus(400);
            return false;
        }
        return true;
 
    }
 
    @ResponseBody
    @GetMapping("/addOne")
    public String addOne(@RequestParam("num") String num) {
        redisBloomFilter.addByBloomFilter(bloomFilterHelper, "bf:device_shadow", String.valueOf(num));
        return num;
    }
}

6.性能测试

• 添加值（add）：

• 查询值（check）：