📕 Redis 相关的业务场景实现
我打算创建一个专栏,主要用于结合八股文和各种场景题来进行代码实践,包括但不限制于集成各种中间件去实现对应的场景和工具封装,该文章为SpringBoot专栏的一个系列,希望大家观看以后帮我多多点赞评论。
大家的点赞和关注是我创作的动力,实属不易,谢谢大家~
本文主要实现了Redis的 发布订阅模式、Steam消息队列、缓存击穿、穿透、雪崩;Redis延迟双删和监听Binlog的业务场景解决方案
⚪ 发布/订阅模式
-
RedisConfig.java
@Bean("messageListenerAdapter") public MessageListenerAdapter messageListenerAdapter(){ return new MessageListenerAdapter(new MySubcribe()); } @Bean("messageListenerAdapter2") public MessageListenerAdapter messageListenerAdapter2(){ return new MessageListenerAdapter(new MySubcribe2()); } @Bean public RedisMessageListenerContainer redisMessageListenerContainer(@Qualifier("messageListenerAdapter") MessageListenerAdapter messageListenerAdapter, @Qualifier("messageListenerAdapter2") MessageListenerAdapter messageListenerAdapter2, LettuceConnectionFactory factory){ RedisMessageListenerContainer container = new RedisMessageListenerContainer(); container.setConnectionFactory(factory); container.addMessageListener(messageListenerAdapter,new ChannelTopic("channel1")); container.addMessageListener(messageListenerAdapter2,new ChannelTopic("channel2")); return container; }
-
实现MySubscribe监听器
@Component public class MySubcribe implements MessageListener { @Override public void onMessage(Message message, byte[] pattern) { GenericJackson2JsonRedisSerializer serializer = new GenericJackson2JsonRedisSerializer(); String deserialize = Objects.requireNonNull(serializer.deserialize(message.getBody())).toString(); System.out.println("当前接收到的消息:" + deserialize); System.out.println("接收数据:"+message.toString()); System.out.println("订阅频道:"+new String(message.getChannel())); } }
-
实现MySucribe2监听器
@Component public class MySubcribe2 implements MessageListener { @Override public void onMessage(Message message, byte[] pattern) { GenericJackson2JsonRedisSerializer serializer = new GenericJackson2JsonRedisSerializer(); UserEntity userEntity = serializer.deserialize(message.getBody(), UserEntity.class); System.out.println(Objects.requireNonNull(userEntity).getName()); System.out.println("接收数据:"+message.toString()); System.out.println("订阅频道:"+new String(message.getChannel())); } }
-
单元测试
@Test public void redisPublish1(){ String message = "redis发布消息到channel信道"; redisTemplate.convertAndSend("channel1",message); UserEntity userEntity = new UserEntity(); userEntity.setName("yjren"); redisTemplate.convertAndSend("channel2",userEntity); }
⚪ Stream 消息队列
@Test
public void redisStream(){
// 添加消息
redisTemplate.opsForStream().add("stream1", Collections.singletonMap("name", "John"));
// 获取消息数量
System.out.println("获取消息的数量:" + redisTemplate.opsForStream().size("stream1"));
// 查询消息
List<MapRecord<String, Object, Object>> records = redisTemplate.opsForStream().range("stream1", Range.unbounded());
for (MapRecord<String, Object, Object> record : records) {
System.out.println("查询消息 当前的数据结构为:" + record);
}
// 反向查询消息
List<MapRecord<String, Object, Object>> records2 = redisTemplate.opsForStream().reverseRange("stream1", Range.unbounded());
for (MapRecord<String, Object, Object> record : records2) {
System.out.println("反向查询消息 当前的数据结构为:" + record);
}
// 连续读取消息
Map<StreamOffset<String>, ReadOffset> streams = new HashMap<>();
streams.put(StreamOffset.create("stream1", ReadOffset.from("0")), ReadOffset.lastConsumed());
List<MapRecord<String, String, String>> results = redisTemplate.opsForStream().read(streams, Duration.ofMillis(5000));
for (MapRecord<String, String, String> result : results) {
System.out.println(result);
}
// 确认处理消息
redisTemplate.execute(new SessionCallback<>() {
@Override
public <K, V> Object execute(RedisOperations<K, V> operations) throws DataAccessException {
operations.watch("stream1");
RecordId id1 = RecordId.of("1001");
RecordId id2 = RecordId.of("1002");
String key = "stream1";
StreamOperations<String, String, String> ops = operations.opsForStream();
List<MapRecord<String, String, String>> records = ops.range(key, Range.closed(id1, id2));
if (!CollectionUtils.isEmpty(records)) {
RedisConnectionFactory factory = operations.getConnectionFactory();
RedisClientInfo info = factory.getConnection().getClientList().get(0);
ops.acknowledge(key, info.getAddress(), id1, id2);
}
return null;
}
});
// 删除消息
Long count = redisTemplate.opsForStream().delete("stream1", "1001", "1002");
System.out.println(count);
// 管理消费者组
redisTemplate.opsForStream().createGroup("stream1", "consumer1");
// 读取并分配消息
MapRecord<String, String, String> record = redisTemplate.opsForStream().read("consumer1", StreamOffset.lastConsumed("stream1"));
if (record != null) {
System.out.println(record);
redisTemplate.opsForStream().acknowledge("stream1", "consumer1", record.getId());
}
}
⚪ 缓存击穿、穿透、雪崩
-
缓存击穿 (一) 缓存空值
/** * 缓存击穿: 缓存空值 * 缓存空值 * 缓存击穿的一种场景是:我们查询这个数据在数据库始终不存在,所以就没有缓存到redis。那么每次查询的过程都 * 需要走数据库,相当于缓存击穿的效果。 */ @Test public void redisTest0() throws InterruptedException { //redis 填充一个 key , 过期时间 为 十秒。 // 当十秒后,我们重新去获取该值 发现没有数据了,这时候我们也应该把空值缓存到redis中。 redisTemplate.opsForValue().set("yjren:china:shanxi","xiaoru"); redisTemplate.expire("yjren:china:shanxi",10,TimeUnit.SECONDS); /** * 现在是十秒内,数据依然存在, * */ String value = redisTemplate.opsForValue().get("yjren:china:shanxi"); System.out.println("当前的缓存还没失效,value值是:" + value); Thread.sleep(15000); /** * 十秒之后缓存,失效 */ value = redisTemplate.opsForValue().get("yjren:china:shanxi"); System.out.println(value == null ? "当前缓存失效, 缓存击穿开始" : "当前的缓存还没失效,value值是:" + value); // 假设现在value 为null 是从数据库获取的, 代表此刻数据库没有值,我们依然将空值进行缓存 redisTemplate.opsForValue().set("yjren:china:shanxi",value); redisTemplate.expire("yjren:china:shanxi",10,TimeUnit.SECONDS); value = redisTemplate.opsForValue().get("yjren:china:shanxi"); System.out.println(value == null ? "此刻为null,代表数据是正常的, 防止缓存击穿" : "数据异常"); }
测试结果
当前的缓存还没失效,value值是:xiaoru 当前缓存失效, 缓存击穿开始 此刻为null,代表数据是正常的, 防止缓存击穿
-
缓存击穿 (二) 异步定时更新
/** * 异步定时更新 * 缓存击穿的一种场景是:该数据刚好缓存时间到期,此时刚好大批量请求。导致数据缓存击穿。 * 那么:我们可以设置该数据异步定时更新,一定的时间就去数据库查询该数据,进行数据更新。 * @return * @throws InterruptedException */ @PostMapping("/cacheBreak1") @ResponseBody public String cacheBreak1() throws Exception { //redis 填充一个 key , 过期时间 为 十秒。 // 当十秒后,我们重新去获取该值 发现没有数据了,这时候我们也应该把空值缓存到redis中。 redisTemplate.opsForValue().set("yjren:china:shanxi","xiaoru"); redisTemplate.expire("yjren:china:shanxi",10, TimeUnit.SECONDS); String value = redisTemplate.opsForValue().get("yjren:china:shanxi"); System.out.println("当前的缓存还没失效,value值是:" + value); // 根据第九节的 quartz 定时任务来动态添加任务 每 9秒 更新一次缓存的时间 QuartzCreateParam createParam = QuartzCreateParam.builder() .jobClazz("com.yjren.yjrenweb.quartz.CacheRedisKeyJob") .jobName("CacheRedisKeyJob") .jobGroup("redis") .description("job detail create") .cron("0/9 * * * * ? ") .build(); quartzUtils.addJob(createParam); // String value = redisTemplate.opsForValue().get("yjren:china:shanxi"); // System.out.println(value == null ? "此刻为null,代表数据是正常的, 防止缓存击穿" : "数据异常"); return "任雨杰最帅"; }
-
缓存击穿(三) 互斥锁更新数据
/** * 互斥锁 * 因为上面的异步定时更新,可能存在多个线程同时开启定时任务一直更新,但是其实我们只需要一个线程去更新数据 * 所以,这个时候我们只需要增加关于这个互斥锁的使用,让一个线程去更新,其他线程自旋等待数据获取 * @return * @throws InterruptedException */ @PostMapping("/cacheBreak2") @ResponseBody public String cacheBreak2() throws Exception { String result = "任雨杰最帅"; String key = "yjren:china:taiyuan"; //redis 填充一个 key , 过期时间 为 十秒。 // 当十秒后,我们重新去获取该值 发现没有数据了,这时候我们也应该把空值缓存到redis中。 String value = redisTemplate.opsForValue().get(key); try{ // 是否抢占锁成功, 成功以后则进行赋值 if (retryLock(key,"xiaoru")){ //TODO 业务逻辑 redisTemplate.opsForValue().set(key,"xiaoru"); redisTemplate.expire(key,15,TimeUnit.SECONDS); //但是防止成为死锁,独占锁设置为10秒失效, 那么我们这里进行 定时任务添加 每99秒刷新锁失效时间。 QuartzCreateParam createParam = QuartzCreateParam.builder() .jobClazz("com.yjren.yjrenweb.quartz.CacheRedisKeyJob2") .jobName("CacheRedisKeyJob2") .jobGroup("redis") .description("job detail create") .cron("0/9 * * * * ? ") .build(); quartzUtils.addJob(createParam); result = redisTemplate.opsForValue().get(key) + "说: " + "任雨杰最丑"; }else{ result = redisTemplate.opsForValue().get(key) + "说: " + "任雨杰还是挺帅的"; } } finally{ //删除定时任务 QuartzDetailParam build = QuartzDetailParam.builder().jobName("CacheRedisKeyJob2").jobGroup("redis").build(); quartzUtils.deleteJob(build); // "xiaoru" 这个是固定值,但是实际业务中, 比如我们每个登录账号的 唯一值 userId、 id 等 // 防止自己设置的独占锁被其他人进行删除,这里进行校验、同时我们也可以通过lua脚本进行完成 if("xiaoru".equals(redisTemplate.opsForValue().get(key))){ //删除 独占锁 // 这里需要判断是否为自己的锁,不能删除其他的锁。 redisTemplate.opsForValue().getAndDelete(key); } } // String value = redisTemplate.opsForValue().get("yjren:china:shanxi"); // System.out.println(value == null ? "此刻为null,代表数据是正常的, 防止缓存击穿" : "数据异常"); return result; } /** * 重试三次进行判断是否有其他线程已经填充值,没有填充则获取锁进行填充否则进入重试 * @param key * @return */ private boolean retryLock(String key,String value) throws InterruptedException { int index = 3; while (index > 0){ // redisTemplate.opsForValue().get(key) == null 表示 没有其他线程进行赋值, 那么我们去第二步操作进行获取锁进行赋值 if(redisTemplate.opsForValue().get(key) == null){ if (Boolean.TRUE.equals(redisTemplate.opsForValue().setIfAbsent(UNIQUE_KEY + key, value,10,TimeUnit.SECONDS))){ return true; } TimeUnit.SECONDS.sleep(1); index--; } } return false; }
-
缓存击穿(四)----增加布隆过滤器
-
添加Redission框架
<dependency> <groupId>org.redisson</groupId> <artifactId>redisson-spring-boot-starter</artifactId> <version>3.10.3</version> </dependency>
-
增加LettuceConnectionFactory的Bean
/** * * spring-boot-starter-data-redis 和 redission-spring-boot-starter 都有lettuce-core, 这个时候需要我们自己自定义的 lettuceconnectionFactory Bean * */ @Bean public LettuceConnectionFactory lettuceConnectionFactory() { // 连接池配置 GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig(); // 线程池线程最大空闲数 genericObjectPoolConfig.setMaxIdle(maxIdle); // 线程池线程最小空闲数 genericObjectPoolConfig.setMinIdle(minIdle); // 线程池最大线程数 genericObjectPoolConfig.setMaxTotal(maxTotal); // 当连接池已用完时,客户端应该等待获取新连接的最大时间,单位为毫秒 genericObjectPoolConfig.setMaxWaitMillis(timeout); genericObjectPoolConfig.setTestOnBorrow(false); // 因为我这个是redis 集群配置,更多单机配置请自行搜索。 ClusterClientOptions clusterClientOptions = ClusterClientOptions.builder() // 关闭ping .pingBeforeActivateConnection(false) .build(); LettuceClientConfiguration lettuceClientConfiguration = LettucePoolingClientConfiguration.builder() .clientOptions(clusterClientOptions) // 超时时间 .shutdownTimeout(Duration.ofMillis(timeout)) .poolConfig(genericObjectPoolConfig) .build(); String[] serverArray = redisServerNodes.split(","); Set<RedisNode> nodes = new HashSet<RedisNode>(); for (String ipPort : serverArray) { String[] ipAndPort = ipPort.split(":"); nodes.add(new RedisNode(ipAndPort[0].trim(), Integer.parseInt(ipAndPort[1]))); } RedisClusterConfiguration redisClusterConfiguration = new RedisClusterConfiguration(); redisClusterConfiguration.setClusterNodes(nodes); RedisPassword pwd = RedisPassword.of(redisServerPassword); redisClusterConfiguration.setPassword(pwd); LettuceClientConfiguration clientConfig = LettucePoolingClientConfiguration.builder() .poolConfig(genericObjectPoolConfig) .build(); LettuceConnectionFactory factory = new LettuceConnectionFactory(redisClusterConfiguration, clientConfig); // 重新初始化工厂 factory.afterPropertiesSet(); return factory; }
-
增加关于Redission的配置文件
spring.redission.cluster.nodes=redis://192.168.150.23:7001,redis://192.168.150.22:7002,redis://192.168.150.19:7003,redis://192.168.150.23:8001,redis://192.168.150.22:8002,redis://192.168.150.19:8003
-
增加RedissionConfig配置文件
@Slf4j @Configuration public class RedissonConfig { @Value("${spring.redission.cluster.nodes}") private String clusterNodes; @Bean public RedissonClient redissonClient(){ Config config = new Config(); config.useClusterServers().addNodeAddress(clusterNodes.split(",")).setClientName("YjrenClient").setPassword("root@1234"); ClusterServersConfig clusterServersConfig = config.useClusterServers(); return Redisson.create(config); } }
-
BloomFilter的测试配置文件
@Configuration public class BlomFilterConfig { @Resource private RedissonClient redissonClient; @Bean public RBloomFilter<String> bloomFilter(){ //定义一个布隆过滤器,指定布隆过滤器的名称 RBloomFilter<String> bloomFilter = redissonClient.getBloomFilter("bloomtest"); //定义布隆过滤器的大小,以及误差率 bloomFilter.tryInit(100000L,0.003); return bloomFilter; } }
-
增加关于布隆过滤器的测试
/** * 布隆过滤器 * 防止缓存击穿 * 因为上面的异步定时更新,可能存在多个线程同时开启定时任务一直更新,但是其实我们只需要一个线程去更新数据 * 所以,这个时候我们只需要增加关于这个互斥锁的使用,让一个线程去更新,其他线程自旋等待数据获取 * @return * @throws InterruptedException */ @PostMapping("/cacheBreak4") @ResponseBody public String cacheBreak4() throws Exception { String result = "任雨杰最帅"; String key = "yjren:china:taiyuan"; //判断这个布隆过滤器是否有这个元素,没有的话则进行添加。 这个是我们测试后续操作专门的添加的 bloomFilterExists(key); //布隆过滤器存在这个元素则进行后续操作,否则 则禁止。 目前是肯定存在的 因为我们手动塞入的, 那么后续的实际生产过程中, if (!bloomFilter.contains(key)){ result = "暂时没有这个数据,进行数据拦截"; } else { //redis 填充一个 key , 过期时间 为 十秒。 // 当十秒后,我们重新去获取该值 发现没有数据了,这时候我们也应该把空值缓存到redis中。 String value = redisTemplate.opsForValue().get(key); try{ // 是否抢占锁成功, 成功以后则进行赋值 if (retryLock(key,"xiaoru")){ //TODO 业务逻辑 redisTemplate.opsForValue().set(key,"xiaoru"); redisTemplate.expire(key,15,TimeUnit.SECONDS); //但是防止成为死锁,独占锁设置为10秒失效, 那么我们这里进行 定时任务添加 每99秒刷新锁失效时间。 QuartzCreateParam createParam = QuartzCreateParam.builder() .jobClazz("com.yjren.yjrenweb.quartz.CacheRedisKeyJob2") .jobName("CacheRedisKeyJob2") .jobGroup("redis") .description("job detail create") .cron("0/9 * * * * ? ") .build(); quartzUtils.addJob(createParam); result = redisTemplate.opsForValue().get(key) + "说: " + "任雨杰最丑"; }else{ result = redisTemplate.opsForValue().get(key) + "说: " + "任雨杰还是挺帅的"; } } finally{ //删除定时任务 QuartzDetailParam build = QuartzDetailParam.builder().jobName("CacheRedisKeyJob2").jobGroup("redis").build(); quartzUtils.deleteJob(build); // "xiaoru" 这个是固定值,但是实际业务中, 比如我们每个登录账号的 唯一值 userId、 id 等 // 防止自己设置的独占锁被其他人进行删除,这里进行校验、同时我们也可以通过lua脚本进行完成 if("xiaoru".equals(redisTemplate.opsForValue().get(key))){ //删除 独占锁 // 这里需要判断是否为自己的锁,不能删除其他的锁。 redisTemplate.opsForValue().getAndDelete(key); } } // String value = redisTemplate.opsForValue().get("yjren:china:shanxi"); // System.out.println(value == null ? "此刻为null,代表数据是正常的, 防止缓存击穿" : "数据异常"); } return result; } private void bloomFilterExists(String value){ if(!bloomFilter.contains(value)){ bloomFilter.add(value); } }
-
-
缓存穿透和缓存雪崩的解决方案与缓存穿透类似,故不在列举。
⚪ 解决Redis和数据库不一致操作
-
延迟双删
- 自定义注解ClearAndReloadCache
@Retention(RetentionPolicy.RUNTIME) @Documented @Target(ElementType.METHOD) public @interface ClearAndReloadCache { String name() default ""; }
-
切面逻辑
@Aspect public class ClearAndReloadCacheAspect { @Resource private RedisTemplate<String,String> redisTemplate; @Pointcut("@annotation(com.yjren.yjrenweb.annotation.ClearAndReloadCache)") public void pointCut(){ } @Around("pointCut()") public Object aroundAdvice(ProceedingJoinPoint point) throws Exception { MethodSignature signature = (MethodSignature) point.getSignature(); Method method = signature.getMethod(); LocalVariableTableParameterNameDiscoverer parameterNameDiscoverer = new LocalVariableTableParameterNameDiscoverer(); String[] parameterNames = parameterNameDiscoverer.getParameterNames(method); int keyNameIndex = ArrayUtils.indexOf(Objects.requireNonNull(parameterNames), "keyName"); if (keyNameIndex == -1) { throw new Exception("fileName not exist export control not work"); } Object[] args = point.getArgs(); String key =String.valueOf(args[keyNameIndex]); if (Boolean.TRUE.equals(redisTemplate.hasKey(key))) { redisTemplate.delete(key); } //执行加入双删注解的改动数据库的业务 即controller中的方法业务 Object proceed = null; try { proceed = point.proceed(); } catch (Throwable throwable) { throwable.printStackTrace(); } //开一个线程 延迟1秒(此处是1秒举例,可以改成自己的业务) // 在线程中延迟删除 同时将业务代码的结果返回 这样不影响业务代码的执行 new Thread(() -> { try { Thread.sleep(200); if (Boolean.TRUE.equals(redisTemplate.hasKey(key))) { redisTemplate.delete(key); } System.out.println("-----------200毫秒钟后,在线程中延迟删除完毕 -----------"); } catch (InterruptedException e) { e.printStackTrace(); } }).start(); return proceed; } }
-
业务代码逻辑
/** * 延迟双删的过程是指:我们在更新数据库的过程中 一般最稳妥方便的方案: 先删除缓存,再更新数据库。 * 但是这个方案其实是两步操作: * ① 删除缓存 * ② 更新数据库 * 在上面这个操作中,是没有原子性的。导致可能①和②中间可能也存在另一个线程发起数据请求。而数据请求的一般操作为: * ① 查询缓存,有则返回数据 * ② 查询数据库,更新缓存并返回数据。 * 那么在这个过程中,可能就会存在缓存和数据库数据不一致问题。 * 那么,这个时候呢我们的解决方案就是延迟双删策略。 * * 这里必须加keyName 参数, 并且参数名字必须是这个, 因为自定义注解中需要解析该方法参数,然后去redis中根据这个keyName 去删除对应的缓存 * @return */ @GetMapping("/cacheBreak5/{keyName}") @ClearAndReloadCache public String cacheBreak5(@PathVariable("keyName") String keyName){ UserEntity userEntity = new UserEntity(); userEntity.setId(20L); userEntity.setName("西北大花"); userEntity.setPhone("15835125802"); userEntity.setOperatorId("1001"); userEntity.setOperatorName("任雨杰"); log.info("目前更新的key名称是:{}",keyName); // 更新数据库信息 userEntityMapper.updateByPrimaryKey(userEntity); return "任雨杰真帅"; }
-
监听mysql的binlog日志文件进行更新缓存
-
引入依赖
<!-- https://mvnrepository.com/artifact/com.github.shyiko/mysql-binlog-connector-java --> <dependency> <groupId>com.github.shyiko</groupId> <artifactId>mysql-binlog-connector-java</artifactId> <version>0.17.0</version> </dependency> <!-- https://mvnrepository.com/artifact/com.google.guava/guava --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>32.1.2-jre</version> </dependency> <dependency> <groupId>cn.hutool</groupId> <artifactId>hutool-all</artifactId> <version>5.4.1</version> </dependency> <!-- 导入 JDBCTemplate 模板 --> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-jdbc</artifactId> </dependency>
-
jdbcTemplate 注解bean
@Bean public JdbcTemplate jdbcTemplate(@Autowired @Qualifier("master") DataSource masterDataSource){ return new JdbcTemplate(masterDataSource); } /* // @Primary @Bean(name = "master") // @QuartzDataSource @ConfigurationProperties(prefix = "spring.datasource.master") public DataSource masterDataSource(){ return DataSourceBuilder.create().build(); } */ /** spring.datasource.master.driver-class-name=com.mysql.cj.jdbc.Driver spring.datasource.master.jdbc-url=jdbc:mysql://192.168.150.25:3306/rzleyou?useSSL=false&serverTimezone=UTC spring.datasource.master.username=root spring.datasource.master.password=root@1234 */
-
自定义datasource配置信息
custom.datasource.binlogsync.host=192.168.150.25 custom.datasource.binlogsync.port=3306 custom.datasource.binlogsync.username=root custom.datasource.binlogsync.password=root@1234 custom.datasource.binlogsync.database=rzleyou
-
binlog注解Config配置信息
@Data @Configuration @ConfigurationProperties(prefix = "custom.datasource.binlogsync") public class BinlogSyncConfig { private String host; private int port; private String username; private String password; private String database; @Bean public BinaryLogClient binaryLogClient() { BinaryLogClient client = new BinaryLogClient(host, port, database, username, password); EventDeserializer eventDeserializer = new EventDeserializer(); eventDeserializer.setCompatibilityMode( EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG, EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY ); return client; } }
-
监听器接口BinLogListener
public interface BinLogListener { /** * 将监听器注册到处理器中 */ void register(BinlogHandler binlogHandler); /** * 当触发这些事件的时候 * * @param rowData */ void onEvent(BinLogRowData rowData) throws Exception; }
-
BinLogListener接口实现
/** * 配置表binlog监听 * @author Administrator */ @Slf4j @Component("configListener") public class ConfigListener implements BinLogListener { private static final String DATABASE = "rzleyou"; private static final String TABLE = "t_coun_user"; private static final String CONF_KEY = "REDIS:CONF"; @Autowired private RedisTemplate<String,String> redisTemplate; @Resource private UserEntityMapper userEntityMapper; @Override public void register(BinlogHandler binlogHandler) { binlogHandler.register(DATABASE, TABLE, this); } @Override public void onEvent(BinLogRowData rowData) throws Exception { log.info("====> class={}, rowData={}, eventType={}", this.getClass().getName(), rowData, rowData.getType()); switch (rowData.getType()) { case EXT_WRITE_ROWS: case WRITE_ROWS: case EXT_UPDATE_ROWS: case UPDATE_ROWS: case EXT_DELETE_ROWS: case DELETE_ROWS: freshCache(); default: break; } } private void freshCache() { // 查询所有生效的配置 //TODO 进行相关的缓存更新操作 System.out.println("任雨杰是真他娘的nice"); } }
-
关于数据库的表结构工具类封装
@Slf4j @Component public class BingLogTableHolder { @Resource private JdbcTemplate jdbcTemplate; /** * 查询关于表结构的相关字段 */ private static final String INIT_SQL = "select `table_schema`, `table_name`, `column_name`, `ordinal_position` " + "from information_schema.columns " + "where `table_schema` = ? and `table_name` = ?"; private static final Map<String, Map<Integer, String>> TABLE_COLUMN_MAP = new ConcurrentHashMap<>(64); public void initTable(String databaseName, String tableName) { int queryTimeout = jdbcTemplate.getQueryTimeout(); List<TableColumn> query = jdbcTemplate.query(INIT_SQL, new Object[]{databaseName, tableName}, (rs, index) -> new TableColumn().mapRow(rs, index)); Map<Integer, String> collect = query.stream().collect(Collectors.toMap(TableColumn::getPosition, TableColumn::getColumnName)); TABLE_COLUMN_MAP.put(databaseName + ":" + tableName, collect); } public Map<Integer, String> getTable(String databaseName, String tableName) { return TABLE_COLUMN_MAP.get(databaseName + ":" + tableName); } @Getter @Setter public static class TableColumn implements RowMapper<TableColumn> { private String columnName; private Integer position; private String tableName; private String databaseName; @Override public TableColumn mapRow(ResultSet rs, int i) throws SQLException { this.databaseName = rs.getString("table_schema"); this.tableName = rs.getString("table_name"); this.columnName = rs.getString("column_name"); this.position = rs.getInt("ordinal_position"); return this; } } }
-
实现BinLogClient.EventListener的具体实现
@Slf4j @Component public class BinlogHandler implements BinaryLogClient.EventListener { private final Map<String, BinLogListener> listenerMap = new ConcurrentHashMap<>(64); private final Map<Long, String> dbMap = new ConcurrentHashMap<>(64); private final Map<Long, String> tableMap = new ConcurrentHashMap<>(64); @Resource private BingLogTableHolder bingLogTableHolder; private String getKey(String dataBaseName, String tableName) { return dataBaseName + ":" + tableName; } public void register(String dataBaseName, String tableName, BinLogListener binLogListener) { this.listenerMap.put(getKey(dataBaseName, tableName), binLogListener); bingLogTableHolder.initTable(dataBaseName, tableName); } @Override public void onEvent(Event event) { final EventType eventType = event.getHeader().getEventType(); log.info("====> binlog event detected! eventType={}", eventType); if (eventType == EventType.TABLE_MAP) { TableMapEventData data = event.getData(); dbMap.put(data.getTableId(), data.getDatabase()); tableMap.put(data.getTableId(), data.getTable()); } Long tableId = -999L; if (eventType == EventType.UPDATE_ROWS || eventType == EventType.EXT_UPDATE_ROWS) { UpdateRowsEventData data = event.getData(); tableId = data.getTableId(); } if (eventType == EventType.WRITE_ROWS || eventType == EventType.EXT_WRITE_ROWS) { WriteRowsEventData data = event.getData(); tableId = data.getTableId(); } if (eventType == EventType.DELETE_ROWS || eventType == EventType.EXT_DELETE_ROWS) { DeleteRowsEventData data = event.getData(); tableId = data.getTableId(); } String db = dbMap.getOrDefault(tableId, ""); String table = tableMap.getOrDefault(tableId, ""); if (StringUtils.isEmpty(db) || StringUtils.isEmpty(table)) { log.error("====> binlog handler error, dataBaseName or tableName is empty"); return; } // 找出对应表有兴趣的监听器 String key = getKey(db, table); BinLogListener listener = this.listenerMap.get(key); if (null == listener) { return; } log.info("====> trigger event: {}", eventType.name()); try { BinLogRowData rowData = buildRowData(event); if (rowData == null) { return; } rowData.setType(eventType); rowData.setTableName(db); rowData.setDataBaseName(table); listener.onEvent(rowData); } catch (Exception e) { log.error("====> binlog handler error ,error message:{}", e.getMessage(), e); } } private List<Serializable[]> getAfterValues(EventData eventData) { log.info("====> binlog event getAfterValues"); if (eventData instanceof WriteRowsEventData) { return ((WriteRowsEventData) eventData).getRows(); } if (eventData instanceof UpdateRowsEventData) { return ((UpdateRowsEventData) eventData).getRows() .stream() .map(Map.Entry::getValue) .collect(Collectors.toList()); } if (eventData instanceof DeleteRowsEventData) { return null; } return Collections.emptyList(); } private List<Serializable[]> getBeforeValues(EventData eventData) { log.info("====> binlog event getBeforeValues"); if (eventData instanceof WriteRowsEventData) { return null; } if (eventData instanceof UpdateRowsEventData) { return ((UpdateRowsEventData) eventData).getRows() .stream() .map(Map.Entry::getKey) .collect(Collectors.toList()); } if (eventData instanceof DeleteRowsEventData) { return ((DeleteRowsEventData) eventData).getRows(); } return Collections.emptyList(); } private BinLogRowData buildRowData(Event event) { final EventData data = event.getData(); BinLogRowData rowData = new BinLogRowData(); Long tableId = null; if (data instanceof WriteRowsEventData) { WriteRowsEventData writeRowsEventData = (WriteRowsEventData) data; tableId = writeRowsEventData.getTableId(); } else if (data instanceof UpdateRowsEventData) { UpdateRowsEventData updateRowsEventData = (UpdateRowsEventData) data; tableId = updateRowsEventData.getTableId(); } else if (data instanceof DeleteRowsEventData) { DeleteRowsEventData deleteRowsEventData = (DeleteRowsEventData) data; tableId = deleteRowsEventData.getTableId(); } final Map<Integer, String> table = bingLogTableHolder.getTable(dbMap.getOrDefault(tableId, ""), tableMap.getOrDefault(tableId, "")); List<Map<String, Object>> afterMapList = new ArrayList<>(); List<Map<String, Object>> beforeMapList = new ArrayList<>(); final List<Serializable[]> beforeValues = getBeforeValues(data); final List<Serializable[]> afterValues = getAfterValues(data); if (afterValues != null) { for (Serializable[] after : afterValues) { Map<String, Object> afterMap = new HashMap<>(64); for (int j = 0; j < after.length; j++) { String colName = table.get(j + 1); if (colName == null) { continue; } Object afterValue = after[j]; afterMap.put(colName, afterValue); if ("id".equals(colName)) { rowData.setDataId(afterValue); } } afterMapList.add(afterMap); rowData.setAfter(afterMapList); } } if (beforeValues != null) { for (Serializable[] before : beforeValues) { Map<String, Object> beforeMap = new HashMap<>(64); for (int j = 0; j < before.length; j++) { String colName = table.get(j + 1); if (colName == null) { continue; } Object afterValue = before[j]; beforeMap.put(colName, afterValue); if ("id".equals(colName)) { rowData.setDataId(afterValue); } } beforeMapList.add(beforeMap); rowData.setBefore(beforeMapList); } } return rowData; } }
-
BinlogRowData 相关数据库修改的记录
@Slf4j @Getter @Setter @ToString public class BinLogRowData { /** * 数据库的名称 */ private String dataBaseName; /** * 监听到变化表的名称 */ private String tableName; /** * 修改数据库的主键 id */ private Object dataId; /** * 监听的 binglog 类型 */ private EventType type; /** * 修改前的数据 */ private List<Map<String, Object>> after; /** * 修改后的数据 */ private List<Map<String, Object>> before; public <T extends UserEntity> List<T> getAfterData(Class<T> clazz) throws Exception { List<T> result = Lists.newArrayList(); if (after == null) { return result; } for (Map<String, Object> map : after) { result.add(getData(map, clazz)); } return result; } public <T extends UserEntity> List<T> getBeforeData(Class<T> clazz) throws Exception { List<T> result = Lists.newArrayList(); if (before == null) { return result; } for (Map<String, Object> map : before) { result.add(getData(map, clazz)); } return result; } private <T extends UserEntity> T getData(Map<String, Object> data, Class<T> clazz) throws IllegalAccessException, InstantiationException { final T t = clazz.newInstance(); if (data != null) { data.forEach((k, v) -> { final String fieldName = getFieldName(k); Field declaredField = null; try { declaredField = clazz.getDeclaredField(fieldName); } catch (NoSuchFieldException e) { try { declaredField = clazz.getSuperclass().getDeclaredField(fieldName); } catch (NoSuchFieldException e1) { log.warn("====> Field【{}】 not found!", fieldName, e1); } } try { if (declaredField != null) { declaredField.setAccessible(true); declaredField.set(t, v); declaredField.setAccessible(false); } } catch (IllegalAccessException e) { e.printStackTrace(); } }); } return t; } private String getFieldName(String columnName) { final String[] split = columnName.split("_"); final String collect = Stream.of(split) .map(text -> text.substring(0, 1).toUpperCase() + text.substring(1, text.length()) ).collect(Collectors.joining()); return collect.substring(0, 1).toLowerCase() + collect.substring(1, collect.length()); } }
-
BinLogClient 客户端
@Slf4j @Component public class BinLogClient { @Resource private BinaryLogClient binaryLogClient; @Autowired private Map<String, BinLogListener> binLogListenerMap; @Resource private BinlogHandler binlogHandler; private ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(1,1,2L, TimeUnit.SECONDS,new LinkedBlockingDeque<>(100),new ThreadFactoryBuilder().setNamePrefix("binlog-starter-pool- %d").build(),new ThreadPoolExecutor.CallerRunsPolicy()); @PostConstruct public void start(){ final Set<Map.Entry<String, BinLogListener>> entries = binLogListenerMap.entrySet(); for (Map.Entry<String, BinLogListener> entry : entries) { final BinLogListener listener = entry.getValue(); listener.register(binlogHandler); } poolExecutor.execute(()->{ binaryLogClient.registerEventListener(binlogHandler); try{ binaryLogClient.connect(); } catch (Exception e){ log.error("====> binLog client connect error!", e); } }); } @PreDestroy public void destroy() throws Exception { if (binaryLogClient != null) { binaryLogClient.disconnect(); } } }
-