debezium监听数据库变更

debezium是一款可以监听到数据库数据变更的工具，我使用的场景是将数据库数据的增删改记录下来并展示在页面上，我的数据量不大，没有使用到消息队列，大型的系统可以配合消息队列使用，比如说kafka。

1、首先根据自己的java和mysql版本引入合适的依赖，我mysql是8，java8，使用的依赖版本是<debezium.version>1.9.0.Final</debezium.version>

<debezium.version>1.9.0.Final</debezium.version>
        <dependency>
			<groupId>io.debezium</groupId>
			<artifactId>debezium-api</artifactId>
			<version>${debezium.version}</version>
		</dependency>
		<dependency>
			<groupId>io.debezium</groupId>
			<artifactId>debezium-embedded</artifactId>
			<version>${debezium.version}</version>
		</dependency>
		<dependency>
			<groupId>io.debezium</groupId>
			<artifactId>debezium-connector-mysql</artifactId>
			<version>${debezium.version}</version>
		</dependency>

2、配置数据库信息，我是配置在nacos使用配置类去读取的

debezium:
  database:
    - name: 数据库名
      enabled: true
      serverId: 服务ID
      host: 数据库地址
      port: 数据库端口
      username: 数据库用户名
      password: 数据库密码
      offset-path: temp/debezium/test/offsets/offset.dat
      history-path: temp/debezium/test/history/custom-file-db-history.dat
      table:
        要监听的表名

3、读取配置类

@Configuration
@ConfigurationProperties(prefix = "debezium")
public class MysqlConfig {

	private List<DatabaseData> database;

	public void setDatabase(List<DatabaseData> database) {
		this.database = database;
	}

	@Bean
	public List<String> tableList() {
		List<String> list = new ArrayList<>();
		for (DatabaseData dbd : database) {
			list.addAll(dbd.getTable());
		}
		return list;
	}

	@Bean
	public List<Properties> mysqlProperties() {
		List<Properties> list = new ArrayList<>();

		String osName = System.getProperty("os.name");
		String path = "/";
		if (osName.startsWith("Windows")) {
			path = "D:/";
		} else if (osName.startsWith("Linux")) {
			path = "/opt/";
		}

		for (DatabaseData dbd : database) {
			//保存路径重新赋值
			dbd.setOffsetPath(path + dbd.getOffsetPath());
			dbd.setHistoryPath(path + dbd.getHistoryPath());

			Properties props = new Properties();
			props.setProperty("name", dbd.getName());
			props.setProperty("connector.class", "io.debezium.connector.mysql.MySqlConnector");
			props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore");
			props.setProperty("offset.storage.file.filename", dbd.getOffsetPath());
			props.setProperty("offset.flush.interval.ms", "600000");
			props.setProperty("database.hostname", dbd.getHost());
			props.setProperty("database.port", dbd.getPort());
			props.setProperty("database.user", dbd.getUsername());
			props.setProperty("database.password", dbd.getPassword());
			props.setProperty("database.server.id", dbd.getServerId());
			props.setProperty("database.server.name", "my_mysql_connector" + dbd.getName());
			props.setProperty("database.history",
					"io.debezium.relational.history.FileDatabaseHistory");
			props.setProperty("database.history.file.filename", dbd.getHistoryPath());
			String tableList = dbd.getTable().stream().map(item -> item.contains("&") ? item.substring(0, item.indexOf("&")) : item).collect(Collectors.joining(","));
			props.setProperty("table.include.list", tableList);
			props.setProperty("decimal.handling.mode", "string");
			list.add(props);
		}
		return list;
	}

}

4、核心监听类 MysqlListener 

@Component
public class MysqlListener extends AbstractDatabaseChangeListener {


	private MysqlListener(@Qualifier("mysqlProperties") List<Properties> list) {
        //执行父类的构造方法，根据配置激活监听器
		super(list);
	}

	@Override
	public Map<String, Object> dataChangeLogic(String handleType,ChangeData record, String tableName) {

		Map<String, LogicHandler> logicHandlerMap = SpringUtil
				.getBeansOfType(LogicHandler.class);

		Optional<LogicHandler> optional = logicHandlerMap.values().stream()
				.filter(service -> service.support("mysql"))
				.min(Comparator.comparingInt(Ordered::getOrder));

		if (!optional.isPresent()) {
			throw new BizException("LogicHandler error , not register");
		}
		LogicHandler logicHandler = optional.get();
		return logicHandler.handle(handleType,record, tableName);
	}

}

5、

@Slf4j
public abstract class AbstractDatabaseChangeListener implements DBChangeListener, ApplicationListener<DBChangeEvent> {

	@Resource
	protected MessageService messageService;
	private final Object initializationLock = new Object();
	private volatile boolean initialized = false;

	private volatile Boolean needSendMessage = false;

	private final List<DebeziumEngine<ChangeEvent<String, String>>> engineList = new ArrayList<>();

	@Resource
	protected RedisTemplate<String, Object> redisTemplate;


	private final String REDIS_KEY = "data:monitor:open:flag";

	private final Map<String, String> map = new HashMap<>(8);

	private final ExecutorService executorService = new ThreadPoolExecutor(8, 8,
			0L, TimeUnit.MILLISECONDS,
			new LinkedBlockingQueue<>(), r -> {
		Thread thread = new Thread(r);
		thread.setName("debezium-engine");
		return thread;
	});

	private final AtomicLong startTimeMillis = new AtomicLong(0);

	protected AbstractDatabaseChangeListener(List<Properties> list) {
		for (Properties props : list) {
			this.engineList.add(DebeziumEngine.create(Json.class)
					.using(props)
					.notifying(record -> {
                        //消息变更会调方法处理变更消息
						receiveChangeEvent(record.value());
					}).build());
		}
	}


	@Override
	public void receiveChangeEvent(String record) {
		try {
			if (initialized && StrUtil.isNotEmpty(record)) {
				// 处理变更事件的逻辑
				long eventTimestamp = getEventTimestamp(record);

				// 检查事件的时间戳是否大于或等于初始化时间戳
				if (eventTimestamp >= startTimeMillis.get()) {
					Map<String, Object> payload = JSONUtils.getPayload(record);
					String handleType = JSONUtils.getHandleType(payload);
					if (!("NONE".equals(handleType) || "READ".equals(handleType))) {
						ChangeData changeData = JSONUtils.getChangeData(payload);
						Map<String, Object> data = dataChangeLogic(handleType, changeData, String.valueOf(changeData.getSource().get("table")));
						if (needSendMessage) {
							Message build = Message.builder()
									.data(data)
									.dbType(String.valueOf(changeData.getSource().get("connector")))
									.database(String.valueOf(changeData.getSource().get("db")))
									.table(String.valueOf(changeData.getSource().get("table")))
									.handleType(handleType)
									.build();
							sendMessageAsync(build);
						}
					}
				}
			}
		} catch (Exception e) {
			log.error("receiveChangeEvent error", e);
		}
	}


	/**
	 * 处理数据逻辑 由子类实现
	 *
	 * @param handleType 处理类型 insert update delete
	 * @param record     记录
	 * @param tableName  表名
	 * @return 返回处理后的数据
	 */
	protected abstract Map<String, Object> dataChangeLogic(String handleType, ChangeData record, String tableName);


	public void start() {
		String value = UUID.randomUUID().toString();
		map.put(REDIS_KEY, value);
		Boolean openFlag = redisTemplate.opsForValue().setIfAbsent(REDIS_KEY, value);
		if (!openFlag) {
			return;
		}
		synchronized (initializationLock) {
			if (!initialized) {
				// 执行监听初始化步骤
				for (DebeziumEngine<ChangeEvent<String, String>> engine : engineList) {
					executorService.execute(engine);
				}
				startTimeMillis.set(System.currentTimeMillis());
				initialized = true;
				log.info("start debezium engine success");
			}
		}
	}

	@PreDestroy
	public void stop() {
		try {
			for (DebeziumEngine<ChangeEvent<String, String>> engine : engineList) {
				if (engine != null) {
					try {
						log.info("try to stop engine");
						engine.close();
					} catch (Exception e) {
						log.error("Failed to stop engine", e);
					}
				}
			}
		} finally {
			String mapValue = map.get(REDIS_KEY);
			String redisValue = (String) redisTemplate.opsForValue().get(REDIS_KEY);
			log.info("expire redis key,redisValue:{},mapValue:{}", redisValue, mapValue);
			//如果是当前节点的监听，删除redis的key
			if (mapValue != null && mapValue.equals(redisValue)) {
				redisTemplate.expire(REDIS_KEY, 1, TimeUnit.SECONDS);
				log.info("expire redis key success,value:{}", redisValue);
			}
			log.info("stop debezium engine success");
		}
	}

	private void sendMessageAsync(Message message) {
		CompletableFuture.runAsync(() -> {
			// 实际的消息发送逻辑，可能涉及到网络请求等，异步执行
			messageService.sendMessage(message);
		}, executorService);
	}

	@Override
	public void onApplicationEvent(DBChangeEvent dbChangeEvent) {
		start();
		EventContent source = (EventContent) dbChangeEvent.getSource();
		needSendMessage = source.getNeedSendMessage();
	}


	private long getEventTimestamp(String value) {
		return parseEventTimestamp(value);
	}

	private long parseEventTimestamp(String value) {
		// 根据事件格式解析时间戳，这里需要根据您的具体事件格式来实现
		Map<String, Object> payload = JSONUtils.getPayload(value);
		Object timestampValue = payload.get("ts_ms");
		if (timestampValue instanceof Long) {
			return (Long) timestampValue;
		} else if (timestampValue instanceof String) {
			return Long.parseLong((String) timestampValue);
		} else {
			// 如果无法解析时间戳，则返回默认值
			return 0;
		}
	}
}