Java 解析binlog实现数据同步推送(reids stream)

希风雨

已于 2024-01-25 13:14:45 修改

阅读量599

点赞数 8

文章标签： java 数据库 mysql binlog

于 2024-01-25 13:11:32 首次发布

本文链接：https://blog.csdn.net/ycg_x_y/article/details/135841726

版权

接上文的解析binlog使用reids的stream通道将解析的数据进行推送

解析并发送
binlog解析代码发送端代码

接收端

实体类

import com.github.shyiko.mysql.binlog.event.EventType;
import lombok.Data;

import java.io.Serializable;
import java.util.Map;

/**
 * binlog对象
 **/
@Data
public class BinLogItem implements Serializable {
    private String dbTable;
    private EventType eventType;
    private Long timestamp = null;
    private Long serverId = null;
    // 存储字段-之前的值之后的值
    private Map<String, Object> before = null;
    private Map<String, Object> after = null;
    // 存储字段--类型
    private Map<String, Colum> colum = null;

}

import lombok.Data;

/**
 * 字段属性对象
 *

 **/
@Data
public class Colum {

  public int inx;
  public String colName; // 列名
  public String dataType; // 类型
  public String schema; // 数据库
  public String table; // 表

  public Colum(String schema, String table, int idx, String colName, String dataType) {
    this.schema = schema;
    this.table = table;
    this.colName = colName;
    this.dataType = dataType;
    this.inx = idx;
  }

}

redis 工具类

import lombok.AllArgsConstructor;
import org.springframework.data.redis.connection.stream.Record;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.connection.stream.StreamInfo;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;

import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * spring redis 工具类
 **/
@SuppressWarnings(value = {"unchecked", "rawtypes"})
@Component
@AllArgsConstructor
public class RedisService<K, V> {


    public final RedisTemplate<K, V> redisTemplate;


    public Boolean hasKey(final K key) {
        return redisTemplate.hasKey(key);
    }

    public void setCacheObject(final K key, final V value) {
        redisTemplate.opsForValue().set(key, value);
    }

    @Async
    public void setCacheObjectAsync(final K key, final V value) {
        redisTemplate.opsForValue().set(key, value);
    }

    /**
     * 缓存基本的对象，Integer、String、实体类等
     *
     * @param key      缓存的键值
     * @param value    缓存的值
     * @param timeout  时间
     * @param timeUnit 时间颗粒度
     */
    public void setCacheObject(final K key, final V value, final Long timeout, final TimeUnit timeUnit) {
        redisTemplate.opsForValue().set(key, value, timeout, timeUnit);
    }

    /**
     * 设置有效时间
     *
     * @param key     Redis键
     * @param timeout 超时时间
     * @return true=设置成功；false=设置失败
     */
    public boolean expire(final K key, final long timeout) {
        return expire(key, timeout, TimeUnit.SECONDS);
    }

    /**
     * 设置有效时间
     *
     * @param key     Redis键
     * @param timeout 超时时间
     * @param unit    时间单位
     * @return true=设置成功；false=设置失败
     */
    public Boolean expire(final K key, final long timeout, final TimeUnit unit) {
        return redisTemplate.expire(key, timeout, unit);
    }

    /**
     * 获得缓存的基本对象。
     *
     * @param key 缓存键值
     * @return 缓存键值对应的数据
     */
    public V getCacheObject(final K key) {
        ValueOperations<K, V> operation = redisTemplate.opsForValue();
        return operation.get(key);
    }

    /**
     * 删除单个对象
     *
     * @param key
     */
    public Boolean deleteObject(final K key) {
        return redisTemplate.delete(key);
    }

    /**
     * 删除集合对象
     *
     * @param collection 多个对象
     * @return
     */
    public Long deleteObject(final Collection collection) {
        return redisTemplate.delete(collection);
    }

    /**
     * 缓存List数据
     *
     * @param key      缓存的键值
     * @param dataList 待缓存的List数据
     * @return 缓存的对象
     */
    public Long setCacheList(final K key, final List<V> dataList) {
        Long count = redisTemplate.opsForList().rightPushAll(key, dataList);
        return count == null ? 0 : count;
    }

    /**
     * 获得缓存的list对象
     *
     * @param key 缓存的键值
     * @return 缓存键值对应的数据
     */
    public List<V> getCacheList(final K key) {
        return redisTemplate.opsForList().range(key, 0, -1);
    }

    /**
     * 缓存Set
     *
     * @param key     缓存键值
     * @param dataSet 缓存的数据
     * @return 缓存数据的对象
     */
    public long setCacheSet(final K key, final V... dataSet) {
        Long count = redisTemplate.opsForSet().add(key, dataSet);
        return count == null ? 0 : count;
    }

    /**
     * 获得缓存的set
     *
     * @param key
     * @return
     */
    public Set<V> getCacheSet(final K key) {
        return redisTemplate.opsForSet().members(key);
    }


    public String createStreamGroup(final K key, final String group) {
        return redisTemplate.opsForStream().createGroup(key, group);
    }
    public Boolean streamGroupExists(final K key, String groupName) {
        final StreamInfo.XInfoGroups groups = this.redisTemplate.opsForStream().groups(key);
        return groups.stream().anyMatch(b -> Objects.equals(groupName, b.groupName()));
    }

    public RecordId addStreamMap(final K key, final Map<K, V> value) {
        return redisTemplate.opsForStream().add(key, value);
    }

    public RecordId addStreamRecord(final Record<K, ?> record) {
        return redisTemplate.opsForStream().add(record);
    }

    public Long streamACK(final K key, final String group, final String... recordIds) {
        return redisTemplate.opsForStream().acknowledge(key, group, recordIds);
    }

    public Long delStream(final K key, final String... recordIds) {
        return redisTemplate.opsForStream().delete(key, recordIds);
    }


}

线程池

import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.springframework.stereotype.Component;

import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

@Component
public class RedisThreadExecutor extends ThreadPoolExecutor {
    private RedisThreadExecutor() {
        super(Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors(), 200L,
                TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(2048),
                new ThreadFactoryBuilder().setNameFormat("async-stream-consumer-runner-%d").setDaemon(true).build());
    }
}

消费者

import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONReader;
import com.energy.cloud.function.sql.BinLogItem;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.connection.stream.MapRecord;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.stream.StreamListener;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.util.Map;

import static com.energy.cloud.function.config.RedisConfig.group;

@Slf4j
@Component
public class SqlConsumer implements StreamListener<String, MapRecord<String, String, String>> {
    @Resource
    private RedisService<String, Object> redisTemplate;

    @Override
    public void onMessage(MapRecord<String, String, String> message) {

        String stream = message.getStream();
        RecordId id = message.getId();
        Map<String, String> value = message.getValue();
        String item = value.get("item");
        String type = value.get("type");
        if (!type.equals("sub")) {
            try {
                BinLogItem binLogItem = JSON.parseObject(item, BinLogItem.class, JSONReader.Feature.SupportSmartMatch);
            } catch (Exception e) {
                log.error("解析binlog数据失败", e);
            }
        }
        log.info("[自动ack]  接收到一个消息 stream:[{" + stream + "}],id:[{" + id + "}],value:[{" + JSON.toJSONString(value) + "}]");

        redisTemplate.streamACK(stream, group, id.getValue());
        redisTemplate.delStream(stream, id.getValue());

    }
}

redis配置

import lombok.var;
import org.springframework.cache.annotation.CachingConfigurerSupport;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.connection.stream.Consumer;
import org.springframework.data.redis.connection.stream.ReadOffset;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.connection.stream.StreamOffset;
import org.springframework.data.redis.stream.StreamMessageListenerContainer;
import org.springframework.data.redis.stream.Subscription;

import javax.annotation.Resource;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;


/**
 * redis配置
 *
 * @author yg
 */
@Configuration
@EnableCaching
public class RedisConfig extends CachingConfigurerSupport {
    @Resource
    private RedisService<String, Object> redisService;
    @Resource
    private RedisThreadExecutor redisThreadExecutor;

    @Resource
    private SqlConsumer sqlConsumer;


    public final static String key = "testReport";
    public final static String group = "testReport1";
    public final static String user = "8848";

    @Bean
    public Subscription subscription(LettuceConnectionFactory factory) {

        var listenerContainer =
                StreamMessageListenerContainer.create(factory, StreamMessageListenerContainer
                        .StreamMessageListenerContainerOptions
                        .builder()
                        .batchSize(5)
                        .executor(redisThreadExecutor)
                        .pollTimeout(Duration.ofSeconds(1))
                        .build());
        initStream(key, group, user);
        // 手动ask消息
//            Subscription subscription = listenerContainer.receive(Consumer.from(redisMqGroup.getName(), redisMqGroup.getConsumers()[0]),
//                    StreamOffset.create(streamName, ReadOffset.lastConsumed()), new ReportReadMqListener());
        // 自动ask消息
        Subscription subscription = listenerContainer.receiveAutoAck(Consumer.from(group, user),
                StreamOffset.create(key, ReadOffset.lastConsumed()), sqlConsumer);
        listenerContainer.start();
        return subscription;
    }


    private void initStream(String key, String group, String user) {
        boolean hasKey = redisService.hasKey(key);
        if (!hasKey || !redisService.streamGroupExists(key, group)) {
            Map<String, Object> map = new HashMap<>(1);
            map.put("item", user + "创建:" + key + "频道,分组:" + group);
            map.put("type", "sub");
            //创建主题
            RecordId recordId = redisService.addStreamMap(key, map);
            redisService.createStreamGroup(key, group);
            redisService.delStream(key, recordId.getValue());
        }

    }


}