使用 Redis 实现布隆过滤器

---

1. WXISME

https://github.com/wxisme/bloomfilter

2. 引入依赖

<dependency>
    <groupId>com.github.wxisme</groupId>
    <artifactId>bloomfilter</artifactId>
    <version>1.0.0</version>
    <exclusions>
        <exclusion>
            <artifactId>jedis</artifactId>
            <groupId>redis.clients</groupId>
        </exclusion>
    </exclusions>
</dependency>

3. 实现代码

这里的 Redis Cluster 配置请参考：使用 Jedis 配置连接 Redis Cluster

import com.github.wxisme.bloomfilter.bitset.RedisBitSet;
import com.github.wxisme.bloomfilter.common.BloomFilter;
import lombok.extern.slf4j.Slf4j;
import redis.clients.jedis.Client;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPool;
import redis.clients.util.JedisClusterCRC16;

import java.util.*;

/**
 * Redis Cluster 工具类
 *
 * @author wangbo
 * @date 2021/6/29
 */
@Slf4j
public class RedisClusterUtil {

    private RedisClusterUtil() {
    }

    /**
     * 向布隆过滤器中添加目标值
     *
     * @param key                      布隆过滤器的key值
     * @param value                    要添加到布隆过滤器中的值
     * @param falsePositiveProbability 期望的假阳性概率
     * @param expectedNumberOfElements 布隆过滤器中元素的预期数量
     */
    public static void addBloomValue(String key, String value, double falsePositiveProbability, int expectedNumberOfElements) {
        try {
            JedisCluster jedisCluster = JedisClusterManager.getJedis();
            BloomFilter<String> filter = new BloomFilter<>(falsePositiveProbability, expectedNumberOfElements);
            filter.bind(new RedisBitSet(jedisCluster, key));
            filter.add(value);
        } catch (Exception e) {
            log.error("redis cluster add bloom value error", e);
        }
    }


    /**
     * 判断布隆过滤其中是否存在目标值
     *
     * @param key                      布隆过滤器的key值
     * @param value                    目标值
     * @param falsePositiveProbability 期望的假阳性概率
     * @param expectedNumberOfElements 布隆过滤器中元素的预期数量
     * @return 判断结果，true存在，false不存在
     */
    public static boolean containsBloomValue(String key, String value, double falsePositiveProbability, int expectedNumberOfElements) {
        try {
            JedisCluster jedisCluster = JedisClusterManager.getJedis();
            BloomFilter<String> filter = new BloomFilter<>(falsePositiveProbability, expectedNumberOfElements);
            filter.bind(new RedisBitSet(jedisCluster, key));
            return filter.contains(value);
        } catch (Exception e) {
            log.error("redis cluster contains bloom value error", e);
            return false;
        }
    }

    /**
     * key前缀
     */
    private static final String KEY_PREFIX = "";

    /**
     * 删除Redis集群中指定前缀的key
     */
    public static void main(String[] args) {
        long start = System.currentTimeMillis();
        JedisCluster jedis = JedisClusterManager.getJedis();
        //获取集群节点
        Map<String, JedisPool> clusterNodes = jedis.getClusterNodes();
        String keysPattern = KEY_PREFIX + "*";
        long countX = 0;
        //循环遍历集群节点
        for (Map.Entry<String, JedisPool> entry : clusterNodes.entrySet()) {
            Jedis jedisNode = entry.getValue().getResource();
            Client client = jedisNode.getClient();
            log.info("Redis 节点信息: IP = {}, port = {}", client.getHost(), client.getPort());
            //判断当前节点是否为slave，只从master节点进行数据删除
            if (!jedisNode.info("replication").contains("role:slave")) {
                long startTime = System.currentTimeMillis();
                //查询当前节点中所有匹配的key
                Set<String> keys = jedisNode.keys(keysPattern);
                log.info("keys 数量：[{}]", keys.size());
                //这里map的初始化大小总槽位数16384除以集群master节点数，并且还应该是2的次方
                Map<Integer, List<String>> map = new HashMap<>(8192);
                for (String key : keys) {
                    //计算key对应的槽位数slot
                    int slot = JedisClusterCRC16.getSlot(key);
                    //cluster模式执行多key操作的时候，这些key必须在同一个slot上，不然会报:JedisDataException
                    //按slot将key分组
                    if (map.containsKey(slot)) {
                        map.get(slot).add(key);
                    } else {
                        List<String> keyList = new ArrayList<>();
                        keyList.add(key);
                        map.put(slot, keyList);
                    }
                }
                long count = 0;
                for (Map.Entry<Integer, List<String>> mapEntry : map.entrySet()) {
                    List<String> keyList = mapEntry.getValue();
                    Long delNum = jedisNode.del(keyList.toArray(new String[0]));
                    log.info("在槽位[{}]上删除了[{}]个key, keyList : [{}]", mapEntry.getKey(), delNum, keyList);
                    count += delNum;
                    countX += delNum;
                }
                log.info("该节点删除了[{}]个key，耗时[{}]ms", count, System.currentTimeMillis() - startTime);
            }
        }
        log.info("删除前缀为[{}]的key任务结束，一共删除[{}]个key，耗时[{}]ms", KEY_PREFIX, countX, System.currentTimeMillis() - start);
    }
}