接上文的解析binlog使用reids的stream通道将解析的数据进行推送
解析并发送
binlog解析代码发送端代码
接收端
实体类
import com.github.shyiko.mysql.binlog.event.EventType;
import lombok.Data;
import java.io.Serializable;
import java.util.Map;
/**
* binlog对象
**/
@Data
public class BinLogItem implements Serializable {
private String dbTable;
private EventType eventType;
private Long timestamp = null;
private Long serverId = null;
// 存储字段-之前的值之后的值
private Map<String, Object> before = null;
private Map<String, Object> after = null;
// 存储字段--类型
private Map<String, Colum> colum = null;
}
import lombok.Data;
/**
* 字段属性对象
*
**/
@Data
public class Colum {
public int inx;
public String colName; // 列名
public String dataType; // 类型
public String schema; // 数据库
public String table; // 表
public Colum(String schema, String table, int idx, String colName, String dataType) {
this.schema = schema;
this.table = table;
this.colName = colName;
this.dataType = dataType;
this.inx = idx;
}
}
redis 工具类
import lombok.AllArgsConstructor;
import org.springframework.data.redis.connection.stream.Record;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.connection.stream.StreamInfo;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.concurrent.TimeUnit;
/**
* spring redis 工具类
**/
@SuppressWarnings(value = {"unchecked", "rawtypes"})
@Component
@AllArgsConstructor
public class RedisService<K, V> {
public final RedisTemplate<K, V> redisTemplate;
public Boolean hasKey(final K key) {
return redisTemplate.hasKey(key);
}
public void setCacheObject(final K key, final V value) {
redisTemplate.opsForValue().set(key, value);
}
@Async
public void setCacheObjectAsync(final K key, final V value) {
redisTemplate.opsForValue().set(key, value);
}
/**
* 缓存基本的对象,Integer、String、实体类等
*
* @param key 缓存的键值
* @param value 缓存的值
* @param timeout 时间
* @param timeUnit 时间颗粒度
*/
public void setCacheObject(final K key, final V value, final Long timeout, final TimeUnit timeUnit) {
redisTemplate.opsForValue().set(key, value, timeout, timeUnit);
}
/**
* 设置有效时间
*
* @param key Redis键
* @param timeout 超时时间
* @return true=设置成功;false=设置失败
*/
public boolean expire(final K key, final long timeout) {
return expire(key, timeout, TimeUnit.SECONDS);
}
/**
* 设置有效时间
*
* @param key Redis键
* @param timeout 超时时间
* @param unit 时间单位
* @return true=设置成功;false=设置失败
*/
public Boolean expire(final K key, final long timeout, final TimeUnit unit) {
return redisTemplate.expire(key, timeout, unit);
}
/**
* 获得缓存的基本对象。
*
* @param key 缓存键值
* @return 缓存键值对应的数据
*/
public V getCacheObject(final K key) {
ValueOperations<K, V> operation = redisTemplate.opsForValue();
return operation.get(key);
}
/**
* 删除单个对象
*
* @param key
*/
public Boolean deleteObject(final K key) {
return redisTemplate.delete(key);
}
/**
* 删除集合对象
*
* @param collection 多个对象
* @return
*/
public Long deleteObject(final Collection collection) {
return redisTemplate.delete(collection);
}
/**
* 缓存List数据
*
* @param key 缓存的键值
* @param dataList 待缓存的List数据
* @return 缓存的对象
*/
public Long setCacheList(final K key, final List<V> dataList) {
Long count = redisTemplate.opsForList().rightPushAll(key, dataList);
return count == null ? 0 : count;
}
/**
* 获得缓存的list对象
*
* @param key 缓存的键值
* @return 缓存键值对应的数据
*/
public List<V> getCacheList(final K key) {
return redisTemplate.opsForList().range(key, 0, -1);
}
/**
* 缓存Set
*
* @param key 缓存键值
* @param dataSet 缓存的数据
* @return 缓存数据的对象
*/
public long setCacheSet(final K key, final V... dataSet) {
Long count = redisTemplate.opsForSet().add(key, dataSet);
return count == null ? 0 : count;
}
/**
* 获得缓存的set
*
* @param key
* @return
*/
public Set<V> getCacheSet(final K key) {
return redisTemplate.opsForSet().members(key);
}
public String createStreamGroup(final K key, final String group) {
return redisTemplate.opsForStream().createGroup(key, group);
}
public Boolean streamGroupExists(final K key, String groupName) {
final StreamInfo.XInfoGroups groups = this.redisTemplate.opsForStream().groups(key);
return groups.stream().anyMatch(b -> Objects.equals(groupName, b.groupName()));
}
public RecordId addStreamMap(final K key, final Map<K, V> value) {
return redisTemplate.opsForStream().add(key, value);
}
public RecordId addStreamRecord(final Record<K, ?> record) {
return redisTemplate.opsForStream().add(record);
}
public Long streamACK(final K key, final String group, final String... recordIds) {
return redisTemplate.opsForStream().acknowledge(key, group, recordIds);
}
public Long delStream(final K key, final String... recordIds) {
return redisTemplate.opsForStream().delete(key, recordIds);
}
}
线程池
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.springframework.stereotype.Component;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@Component
public class RedisThreadExecutor extends ThreadPoolExecutor {
private RedisThreadExecutor() {
super(Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors(), 200L,
TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(2048),
new ThreadFactoryBuilder().setNameFormat("async-stream-consumer-runner-%d").setDaemon(true).build());
}
}
消费者
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONReader;
import com.energy.cloud.function.sql.BinLogItem;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.connection.stream.MapRecord;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.stream.StreamListener;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.Map;
import static com.energy.cloud.function.config.RedisConfig.group;
@Slf4j
@Component
public class SqlConsumer implements StreamListener<String, MapRecord<String, String, String>> {
@Resource
private RedisService<String, Object> redisTemplate;
@Override
public void onMessage(MapRecord<String, String, String> message) {
String stream = message.getStream();
RecordId id = message.getId();
Map<String, String> value = message.getValue();
String item = value.get("item");
String type = value.get("type");
if (!type.equals("sub")) {
try {
BinLogItem binLogItem = JSON.parseObject(item, BinLogItem.class, JSONReader.Feature.SupportSmartMatch);
} catch (Exception e) {
log.error("解析binlog数据失败", e);
}
}
log.info("[自动ack] 接收到一个消息 stream:[{" + stream + "}],id:[{" + id + "}],value:[{" + JSON.toJSONString(value) + "}]");
redisTemplate.streamACK(stream, group, id.getValue());
redisTemplate.delStream(stream, id.getValue());
}
}
redis配置
import lombok.var;
import org.springframework.cache.annotation.CachingConfigurerSupport;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.connection.stream.Consumer;
import org.springframework.data.redis.connection.stream.ReadOffset;
import org.springframework.data.redis.connection.stream.RecordId;
import org.springframework.data.redis.connection.stream.StreamOffset;
import org.springframework.data.redis.stream.StreamMessageListenerContainer;
import org.springframework.data.redis.stream.Subscription;
import javax.annotation.Resource;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
/**
* redis配置
*
* @author yg
*/
@Configuration
@EnableCaching
public class RedisConfig extends CachingConfigurerSupport {
@Resource
private RedisService<String, Object> redisService;
@Resource
private RedisThreadExecutor redisThreadExecutor;
@Resource
private SqlConsumer sqlConsumer;
public final static String key = "testReport";
public final static String group = "testReport1";
public final static String user = "8848";
@Bean
public Subscription subscription(LettuceConnectionFactory factory) {
var listenerContainer =
StreamMessageListenerContainer.create(factory, StreamMessageListenerContainer
.StreamMessageListenerContainerOptions
.builder()
.batchSize(5)
.executor(redisThreadExecutor)
.pollTimeout(Duration.ofSeconds(1))
.build());
initStream(key, group, user);
// 手动ask消息
// Subscription subscription = listenerContainer.receive(Consumer.from(redisMqGroup.getName(), redisMqGroup.getConsumers()[0]),
// StreamOffset.create(streamName, ReadOffset.lastConsumed()), new ReportReadMqListener());
// 自动ask消息
Subscription subscription = listenerContainer.receiveAutoAck(Consumer.from(group, user),
StreamOffset.create(key, ReadOffset.lastConsumed()), sqlConsumer);
listenerContainer.start();
return subscription;
}
private void initStream(String key, String group, String user) {
boolean hasKey = redisService.hasKey(key);
if (!hasKey || !redisService.streamGroupExists(key, group)) {
Map<String, Object> map = new HashMap<>(1);
map.put("item", user + "创建:" + key + "频道,分组:" + group);
map.put("type", "sub");
//创建主题
RecordId recordId = redisService.addStreamMap(key, map);
redisService.createStreamGroup(key, group);
redisService.delStream(key, recordId.getValue());
}
}
}