springboot-redis管道pipline批量新增删除以及scan扫描
本文使用springboot2.1.4的spring-boot-starter-data-redis
jdk1.8;
redis-6.0.3
pom.xml
依赖spring-boot-starter-data-redis
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.4.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.example</groupId>
<artifactId>springboot-redis</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>springboot-redis</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
application.properties
# Redis数据库索引(默认为0)
spring.redis.database=8
# Redis服务器地址
spring.redis.host=10.128.162.142
#spring.redis.cluster.nodes=127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005
# Redis服务器连接端口
spring.redis.port=6379
# Redis服务器连接密码(默认为空)
#spring.redis.password=Anfang@123!
# 连接池最大连接数(使用负值表示没有限制)
spring.redis.jedis.pool.max-active=30
# 连接池最大阻塞等待时间(使用负值表示没有限制)
spring.redis.jedis.pool.max-wait=-1
# 连接池中的最大空闲连接
spring.redis.jedis.pool.max-idle=200
# 连接池中的最小空闲连接
spring.redis.jedis.pool.min-idle=0
# 连接超时时间(毫秒)
spring.redis.timeout=5000
# 看门狗时间,到时还没释放则延长
spring.redis.lock-watchdog-timeout=100
server.port=8002
测试控制器
批量操作每秒可达数万key;
import com.example.springbootredis.RedisUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.*;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import java.io.Serializable;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* redis测试
*/
@Slf4j
@RestController
public class TestController {
final Logger logger = LoggerFactory.getLogger(this.getClass());
@Autowired
private StringRedisTemplate stringRedisTemplate;
@Autowired
private RedisUtil redisUtil;
ThreadPoolExecutor pool = new ThreadPoolExecutor(2, 2, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(1024), (worker, executor) -> {
try {
executor.getQueue().put(worker);
} catch (InterruptedException e) {
e.printStackTrace();
}
});
/**
* 1.批量新增
*/
@GetMapping("saveString")
public String saveString(HttpServletRequest request) {
new Thread(() -> {
for (int n = 1; n < 20; n++) {
HashMap<String, String> datas = new HashMap<>();
for (int i = 0; i < 50000; i++) {
String uuid = UUID.randomUUID().toString().replace("-", "");
datas.put("tom:" + uuid, uuid + uuid + "123456");
}
redisUtil.executePipelined(datas, 100);
logger.info("第{}万", n * 5);
}
}).start();
return "success";
}
/**
* 2.模糊批量删除
* @return
*/
@RequestMapping("deleteAll")
public String delete() {
while (true) {
log.info("第一次扫描开始");
long cursorId = redisUtil.scanAndDelete("tom*");
if (cursorId == 0) {
//判断一下,防止没删除干净
break;
}
log.info("第一次扫描结束");
}
return "success";
}
}
redis自定义工具类
主要功能:通过管道方式批量新增,批量删除;scan扫描并删除;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Slf4j
@Component
public class RedisUtil {
@Autowired
private StringRedisTemplate stringRedisTemplate;
/**
* 管道批量新增
*/
public void executePipelined(Map<String, String> map, long seconds) {
RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
stringRedisTemplate.executePipelined((RedisCallback<String>) connection -> {
map.forEach((key, value) -> {
connection.set(serializer.serialize(key), serializer.serialize(value));
});
return null;
}, serializer);
}
/**
* 管道批量删除
*
* @param keys
*/
public void executeDeletePipelined(List<String> keys) {
log.info("删除数量={}", keys.size());
RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
stringRedisTemplate.executePipelined((RedisCallback<String>) connection -> {
keys.forEach(key -> {
connection.del(serializer.serialize(key));
});
return null;
}, serializer);
}
/**
* 扫描并删除
*
* @param pattern
* @return
*/
public long scanAndDelete(String pattern) {
long start = System.currentTimeMillis();
long cursorId = this.stringRedisTemplate.execute((RedisConnection connection) -> {
//这里传了count的值不管用,依然会遍历出所有符合条件的key
try (Cursor<byte[]> cursor = connection.scan(ScanOptions.scanOptions().match(pattern).build())) {
ArrayList<String> list = new ArrayList<>();
while (cursor.hasNext()) {
list.add(new String(cursor.next(), StandardCharsets.UTF_8));
if (list.size() == 10000) {
//批量删除
this.executeDeletePipelined(list);
list = new ArrayList<>();
}
}
if (list.size() > 0) {
//批量删除
this.executeDeletePipelined(list);
}
return cursor.getCursorId();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
});
log.info("本次删除耗时={}", System.currentTimeMillis() - start);
return cursorId;
}
/**
* 管道批量查询
*
* @param keys
*/
public void executPipelineGet(Collection<String> keys) {
log.info("查询数量={}", keys.size());
RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
final List<Object> objects = stringRedisTemplate.executePipelined((RedisCallback<String>) connection -> {
keys.forEach(key -> {
connection.get(serializer.serialize(key));
});
return null;//这里必须返回null
}, serializer);
final List<String> collect = objects.stream().map(x -> x.toString()).collect(Collectors.toList());
log.info("pip批量查询={}", collect);
}