springboot-redis管道pipline批量新增删除以及scan扫描

springboot-redis管道pipline批量新增删除以及scan扫描
本文使用springboot2.1.4的spring-boot-starter-data-redis
jdk1.8;
redis-6.0.3

pom.xml

依赖spring-boot-starter-data-redis

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.1.4.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.example</groupId>
    <artifactId>springboot-redis</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>springboot-redis</name>
    <description>Demo project for Spring Boot</description>

    <properties>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-redis</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>

</project>

application.properties

# Redis数据库索引（默认为0）
spring.redis.database=8
# Redis服务器地址
spring.redis.host=10.128.162.142
#spring.redis.cluster.nodes=127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005
# Redis服务器连接端口
spring.redis.port=6379
# Redis服务器连接密码（默认为空）
#spring.redis.password=Anfang@123!
# 连接池最大连接数（使用负值表示没有限制）
spring.redis.jedis.pool.max-active=30

# 连接池最大阻塞等待时间（使用负值表示没有限制）
spring.redis.jedis.pool.max-wait=-1
# 连接池中的最大空闲连接
spring.redis.jedis.pool.max-idle=200
# 连接池中的最小空闲连接
spring.redis.jedis.pool.min-idle=0
# 连接超时时间（毫秒）
spring.redis.timeout=5000
# 看门狗时间，到时还没释放则延长
spring.redis.lock-watchdog-timeout=100
server.port=8002

测试控制器

批量操作每秒可达数万key;

import com.example.springbootredis.RedisUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.*;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import javax.servlet.http.HttpServletRequest;
import java.io.Serializable;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * redis测试
 */
@Slf4j
@RestController
public class TestController {
    final Logger logger = LoggerFactory.getLogger(this.getClass());
    @Autowired
    private StringRedisTemplate stringRedisTemplate;
    @Autowired
    private RedisUtil redisUtil;

    ThreadPoolExecutor pool = new ThreadPoolExecutor(2, 2, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(1024), (worker, executor) -> {
        try {
            executor.getQueue().put(worker);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    });

    /**
     * 1.批量新增
     */
    @GetMapping("saveString")
    public String saveString(HttpServletRequest request) {

        new Thread(() -> {

            for (int n = 1; n < 20; n++) {
                HashMap<String, String> datas = new HashMap<>();
                for (int i = 0; i < 50000; i++) {
                    String uuid = UUID.randomUUID().toString().replace("-", "");
                    datas.put("tom:" + uuid, uuid + uuid + "123456");

                }
                redisUtil.executePipelined(datas, 100);

                logger.info("第{}万", n * 5);
            }
        }).start();

        return "success";
    }

    /**
     * 2.模糊批量删除
     * @return
     */
    @RequestMapping("deleteAll")
    public String delete() {
        while (true) {
            log.info("第一次扫描开始");
            long cursorId = redisUtil.scanAndDelete("tom*");
            if (cursorId == 0) {
                //判断一下，防止没删除干净
                break;
            }
            log.info("第一次扫描结束");
        }
        return "success";
    }
}    

redis自定义工具类

主要功能：通过管道方式批量新增，批量删除；scan扫描并删除；

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.stereotype.Component;

import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

@Slf4j
@Component
public class RedisUtil {

    @Autowired
    private StringRedisTemplate stringRedisTemplate;

    /**
     * 管道批量新增
     */
    public void executePipelined(Map<String, String> map, long seconds) {
        RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
        stringRedisTemplate.executePipelined((RedisCallback<String>) connection -> {
            map.forEach((key, value) -> {
                connection.set(serializer.serialize(key), serializer.serialize(value));
            });
            return null;
        }, serializer);
    }

    /**
     * 管道批量删除
     *
     * @param keys
     */
    public void executeDeletePipelined(List<String> keys) {
        log.info("删除数量={}", keys.size());
        RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
        stringRedisTemplate.executePipelined((RedisCallback<String>) connection -> {
            keys.forEach(key -> {
                connection.del(serializer.serialize(key));
            });
            return null;
        }, serializer);
    }

    /**
     * 扫描并删除
     *
     * @param pattern
     * @return
     */
    public long scanAndDelete(String pattern) {
        long start = System.currentTimeMillis();
        long cursorId = this.stringRedisTemplate.execute((RedisConnection connection) -> {
            //这里传了count的值不管用，依然会遍历出所有符合条件的key
            try (Cursor<byte[]> cursor = connection.scan(ScanOptions.scanOptions().match(pattern).build())) {
                ArrayList<String> list = new ArrayList<>();
                while (cursor.hasNext()) {
                    list.add(new String(cursor.next(), StandardCharsets.UTF_8));
                    if (list.size() == 10000) {
                        //批量删除
                        this.executeDeletePipelined(list);
                        list = new ArrayList<>();
                    }
                }
                if (list.size() > 0) {
                    //批量删除
                    this.executeDeletePipelined(list);
                }
                return cursor.getCursorId();
            } catch (Exception e) {
                e.printStackTrace();
                throw new RuntimeException(e);
            }
        });
        log.info("本次删除耗时={}", System.currentTimeMillis() - start);
        return cursorId;
    }
    /**
     * 管道批量查询
     *
     * @param keys
     */
    public void executPipelineGet(Collection<String> keys) {
        log.info("查询数量={}", keys.size());
        RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
        final List<Object> objects = stringRedisTemplate.executePipelined((RedisCallback<String>) connection -> {
            keys.forEach(key -> {
                connection.get(serializer.serialize(key));
            });
            return null;//这里必须返回null
        }, serializer);
        final List<String> collect = objects.stream().map(x -> x.toString()).collect(Collectors.toList());
        log.info("pip批量查询={}", collect);
    }