基于StringRedisTemplate的对pipeline对大量数据的操作
1,添加到 pom.xml 文件
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
2,配置文件的参数:
spring.redis.database=0
spring.redis.host=127.0.0.1
spring.redis.port=6379
3,自定义序列化器(可选)
@Configuration
public class RedisConfig {
@Bean
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory connectionFactory) {
RedisTemplate<String, Object> template = new RedisTemplate<>();
template.setConnectionFactory(connectionFactory);
// 使用StringRedisSerializer来序列化和反序列化redis的key值
template.setKeySerializer(new StringRedisSerializer());
// 使用Jackson2JsonRedisSerializer来序列化和反序列化redis的value值
Jackson2JsonRedisSerializer<Object> jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<>(Object.class);
ObjectMapper om = new ObjectMapper();
// 指定要序列化的域,field, get和set, 以及修饰符范围,任何级别包括private和public
om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
// 指定序列化输入的类型,类必须是非final修饰的,final修饰的类,比如String,Integer等会抛出异常
om.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL);
jackson2JsonRedisSerializer.setObjectMapper(om);
template.setValueSerializer(jackson2JsonRedisSerializer);
// 如果hash中有key和value,此时再指定一个hash的key和value的序列化方式
template.setHashKeySerializer(new StringRedisSerializer());
template.setHashValueSerializer(jackson2JsonRedisSerializer);
template.afterPropertiesSet();
return template;
}
}
4,插入一万条测试数据
当使用pipelined后,向本地插入一万条数据只用时1秒,相比for循环逐条插入快了5倍
@Test
void addDataPipelined(){
// 开始 Pipelining
redisTemplate.executePipelined((RedisCallback<Void>) connection -> {
for (int i = 0; i < 10000; i++) {
String key = "test" + String.format("%04d", i + 1);
HashMap<String, String> data = new HashMap<>();
data.put("key1", "value1");
data.put("key2", "value2");
data.put("key3", "value3");
// 将数据批量插入到 Redis 中
connection.hMSet(key.getBytes(), data.entrySet().stream()
.collect(Collectors.toMap(
entry -> entry.getKey().getBytes(),
entry -> entry.getValue().getBytes()
)));
}
return null;
});
}
5,使用Pipeline查询数据
@SpringBootTest
public class queryRedis {
@Autowired
private StringRedisTemplate redisTemplate;
@Test
void query() {
//1.准备工作
int batchSize = 100;
String prefix = "test";
List<Map<String, Object>> result = new ArrayList<>();
ExecutorService executorService = Executors.newFixedThreadPool(4);
//2.获取所有匹配的key
Set<String> keys = redisTemplate.keys(prefix + "*");
//3.抽取方法,分组处理
List<List<String>> batches = partition(keys, batchSize);
//4.通道执行查询
redisTemplate.executePipelined((RedisCallback<Object>) connection -> {
HashOperations<String, Object, Object> hashOps = redisTemplate.opsForHash();
List<Runnable> tasks = new ArrayList<>();
for (List<String> batch : batches){
tasks.add( () -> {
for (String key : batch){
Map<Object, Object> entries = hashOps.entries(key);
Map<String, Object> stringMap = new LinkedHashMap<>();
//处理数据
for (Map.Entry<Object, Object> entry : entries.entrySet()){
stringMap.put(entry.getKey().toString(), entry.getValue().toString());
}
synchronized (result){
result.add(stringMap);
}
}
});
}
tasks.forEach(executorService::submit);
//关闭线程池
executorService.shutdown();
try{
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
} catch (InterruptedException e){
Thread.currentThread().interrupt();
}
return null;
});
}
//将集合分组为多个子列表,每个子列表包含指定数量的元素
private static <T> List<List<T>> partition(Collection<T> keys, int batchSize) {
List<List<T>> partitions = new ArrayList<>();
Iterator<T> iterator = keys.iterator();
while(iterator.hasNext()){
List<T> batch = new ArrayList<>();
for (int i = 0; i < batchSize && iterator.hasNext(); i++) {
batch.add(iterator.next());
}
partitions.add(batch);
}
return partitions;
}
}
注意:synchronized (result)的作用为防止数据覆盖丢失
如果没有加锁,可能发生以下情况:
线程 A 开始执行:
results 当前状态:[map1]
线程 A 准备添加 stringMapA。
线程 B 开始执行:
results 当前状态:[map1]
线程 B 准备添加 stringMapB。
线程 A 完成添加:
results 变为:[map1, stringMapA]
线程 B 完成添加:
results 变为:[map1, stringMapB]
最终结果可能是 results 列表中只包含了 stringMapB,而 stringMapA 被覆盖或丢失。