Canal + RocketMQ 同步 MySQL 数据到 Redis

解决缓存和数据库一致性问题

一般来说，缓存中的数据没什么问题，但是数据库更新后，就容易出现缓存（Redis）和数据库（MySQL）间的数据一致性问题。由于写和读是并发的，没法保证顺序，就会出现缓存和数据库的数据不一致的问题

Canal工作原理

1. canal模拟mysql slave的交互协议，伪装自己为mysql slave，向mysql master发送dump协议
2. mysql master收到dump请求，开始推送binary log给slave(也就是canal)
3. canal解析binary log对象(原始为byte流)

        Canal的安装与配置（略）

Java操作开始

pom依赖

 <dependency>
            <groupId>org.apache.rocketmq</groupId>
            <artifactId>rocketmq-spring-boot-starter</artifactId>
            <version>2.2.3</version>
            <exclusions>
                <exclusion>
                    <groupId>com.alibaba</groupId>
                    <artifactId>fastjson</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>javax.persistence</groupId>
            <artifactId>persistence-api</artifactId>
            <version>1.0.2</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus-boot-starter</artifactId>
            <version>3.5.1</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba.otter</groupId>
            <artifactId>canal.client</artifactId>
            <version>1.1.4</version>
        </dependency>

Article.java

import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.extension.activerecord.Model;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.Accessors;

/**
 * @ClassName Article
 * @Description TODO
 * @Author fan
 * @Date 2024/1/29 11:59
 * @Version 1.0
 */
@Data
@EqualsAndHashCode(callSuper = false)
@Accessors(chain = true)
public class Article extends Model<Article> {


    @TableId(value = "id", type = IdType.AUTO)
    private Long id;

    private String content;

}

SqlType.enum

/**
 * @ClassName SqlType Canal监听SQL类型
 * @Description TODO
 * @Author fan
 * @Date 2024/1/29 11:50
 * @Version 1.0
 */
@SuppressWarnings("AlibabaEnumConstantsMustHaveComment")
public enum SqlType {

    INSERT("INSERT", "插入"),
    UPDATE("UPDATE", "更新"),
    DELETE("DELETE", "删除");

    private final String type;
    private final String name;

    SqlType(String type, String name) {
        this.type = type;
        this.name = name;
    }

    public String getType() {
        return this.type;
    }

    public String getName() {
        return this.name;
    }

}

定义接口CanalSyncService

package com.example.myrocketmqboot.service;

import com.alibaba.otter.canal.protocol.FlatMessage;

import java.util.Collection;

/**
 * @ClassName CanalSyncService Canal同步服务
 * @Description TODO
 * @Author fan
 * @Date 2024/1/29 14:13
 * @Version 1.0
 */
public interface CanalSyncService<T> {

    /**
     * 处理数据
     *
     * @param flatMessage CanalMQ数据
     */
    void process(FlatMessage flatMessage);

    /**
     * DDL语句处理
     *
     * @param flatMessage CanalMQ数据
     */
    void ddl(FlatMessage flatMessage);

    /**
     * 插入
     *
     * @param list 新增数据
     */
    void insert(Collection<T> list);

    /**
     * 更新
     *
     * @param list 更新数据
     */
    void update(Collection<T> list);

    /**
     * 删除
     *
     * @param list 删除数据
     */
    void delete(Collection<T> list);

}

ArticleCanalRocketMqRedisService实现CanalSyncService接口

package com.example.myrocketmqboot.service.impl;

import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.annotation.TableId;
import com.example.myrocketmqboot.listener.SqlType;
import com.example.myrocketmqboot.service.CanalSyncService;
import lombok.extern.slf4j.Slf4j;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.google.common.collect.Sets;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.util.ReflectionUtils;

import javax.annotation.Resource;
import java.lang.reflect.Field;
import java.lang.reflect.ParameterizedType;
import java.util.*;

/**
 * @ClassName AbstractCanalRocketMqRedisService 抽象Canal-RocketMQ通用处理服务
 * @Description TODO
 * @Author fan
 * @Date 2024/1/29 14:14
 * @Version 1.0
 */
@Slf4j
public abstract class ArticleCanalRocketMqRedisService<T> implements CanalSyncService<T> {

    @Resource
    private RedisTemplate redisTemplate;

    @Resource
    private com.fan.li.myspringboot.util.RedisClient redisUtils;

    private Class<T> classCache;

    /**
     * 获取Model名称
     *
     * @return Model名称
     */
    protected abstract String getModelName();

    /**
     * 处理数据
     *
     * @param flatMessage CanalMQ数据
     */
    @Override
    public void process(FlatMessage flatMessage) {
        if (flatMessage.getIsDdl()) {
            ddl(flatMessage);
            return;
        }

        Set<T> data = getData(flatMessage);

        if (SqlType.INSERT.getType().equals(flatMessage.getType())) {
            insert(data);
        }

        if (SqlType.UPDATE.getType().equals(flatMessage.getType())) {
            update(data);
        }

        if (SqlType.DELETE.getType().equals(flatMessage.getType())) {
            delete(data);
        }

    }

    /**
     * DDL语句处理
     *
     * @param flatMessage CanalMQ数据
     */
    @Override
    public void ddl(FlatMessage flatMessage) {
    }

    /**
     * 插入
     *
     * @param list 新增数据
     */
    @Override
    public void insert(Collection<T> list) {
        insertOrUpdate(list);
    }

    /**
     * 更新
     *
     * @param list 更新数据
     */
    @Override
    public void update(Collection<T> list) {
        insertOrUpdate(list);
    }

    /**
     * 删除
     *
     * @param list 删除数据
     */
    @Override
    public void delete(Collection<T> list) {
        Set<String> keys = Sets.newHashSetWithExpectedSize(list.size());
        for (T data : list) {
            keys.add(getWrapRedisKey(data));
        }
        for (String key : keys) {
            redisUtils.deleteKey(key);
        }
    }

    /**
     * 插入或者更新redis
     *
     * @param list 数据
     */
    @SuppressWarnings("unchecked")
    private void insertOrUpdate(Collection<T> list) {
        redisTemplate.executePipelined((RedisConnection redisConnection) -> {
            for (T data : list) {
                String key = getWrapRedisKey(data);
                // 序列化key
                byte[] redisKey = redisTemplate.getKeySerializer().serialize(key);
                // 序列化value
                byte[] redisValue = redisTemplate.getValueSerializer().serialize(data);
                redisConnection.set(Objects.requireNonNull(redisKey), Objects.requireNonNull(redisValue));
            }
            return null;
        });
    }

    /**
     * 封装redis的key
     *
     * @param t 原对象
     * @return key
     */
    protected String getWrapRedisKey(T t) {
        return getModelName() + ":" + getIdValue(t);
    }

    /**
     * 获取类泛型
     *
     * @return 泛型Class
     */
    @SuppressWarnings("unchecked")
    protected Class<T> getTypeArgument() {
        if (classCache == null) {
            classCache = (Class) ((ParameterizedType) this.getClass().getGenericSuperclass()).getActualTypeArguments()[0];
        }
        return classCache;
    }

    /**
     * 获取Object标有@TableId注解的字段值
     *
     * @param t 对象
     * @return id值
     */
    protected Object getIdValue(T t) {
        Field fieldOfId = getIdField();
        ReflectionUtils.makeAccessible(fieldOfId);
        return ReflectionUtils.getField(fieldOfId, t);
    }

    /**
     * 获取Class标有@TableId注解的字段名称
     *
     * @return id字段名称
     */
    protected Field getIdField() {
        Class<T> clz = getTypeArgument();
        Field[] fields = clz.getDeclaredFields();
        for (Field field : fields) {
            TableId annotation = field.getAnnotation(TableId.class);

            if (annotation != null) {
                return field;
            }
        }
        log.error("PO类未设置@TableId注解");
        return null;
    }

    /**
     * 转换Canal的FlatMessage中data成泛型对象
     *
     * @param flatMessage Canal发送MQ信息
     * @return 泛型对象集合
     */
    protected Set<T> getData(FlatMessage flatMessage) {
        List<Map<String, String>> sourceData = flatMessage.getData();
        Set<T> targetData = Sets.newHashSetWithExpectedSize(sourceData.size());
        for (Map<String, String> map : sourceData) {
            T t = JSON.parseObject(JSON.toJSONString(map), getTypeArgument());
            targetData.add(t);
        }
        return targetData;
    }
}

ArticleConsumer消费

package com.example.myrocketmqboot.consumer.article;

import com.alibaba.otter.canal.protocol.FlatMessage;
import com.example.myrocketmqboot.pojo.Article;
import com.example.myrocketmqboot.service.impl.AbstractCanalRocketMqRedisService;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.apache.rocketmq.spring.core.RocketMQPushConsumerLifecycleListener;
import org.springframework.stereotype.Service;

/**
 * @ClassName ArticleConsumer
 * @Description TODO
 * @Author fan
 * @Date 2024/1/29 14:19
 * @Version 1.0
 */
@Slf4j
@Service
@RocketMQMessageListener(topic = "Consumer_Article", consumerGroup = "myArticle")
public class ArticleConsumer extends AbstractCanalRocketMqRedisService<Article>
        implements RocketMQListener<FlatMessage>, RocketMQPushConsumerLifecycleListener {

    @Override
    public void onMessage(FlatMessage flatMessage) {
        log.info("consumer message {}", flatMessage);
        try {
            process(flatMessage);
        } catch (Exception e) {
            log.warn(String.format("[onMessage_消费失败]", flatMessage), e);
            throw new RuntimeException(e);
        }
    }

    @Getter
    private final String modelName = Article.class.getSimpleName();

    @Override
    public void prepareStart(DefaultMQPushConsumer consumer) {
        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
        consumer.setConsumeTimestamp(UtilAll.timeMillisToHumanString3(System.currentTimeMillis()));
    }
}

测试：

1、往article表插入一条数据

2、查看redis有该条数据即可！