数据同步-debezium内嵌至springboot

a179583637

已于 2024-07-31 11:23:44 修改

阅读量467

点赞数 3

文章标签： spring boot 数据库大数据数据仓库

于 2024-06-12 17:29:35 首次发布

本文链接：https://blog.csdn.net/a179583637/article/details/139626366

版权

一、官方网站

很重要，必看，尤其是各个连接器的配置

Debezium

二、引入依赖

<!-- https://mvnrepository.com/artifact/io.debezium/debezium-embedded -->
        <dependency>
            <groupId>io.debezium</groupId>
            <artifactId>debezium-embedded</artifactId>
            <version>2.5.1.Final</version>
        </dependency>

<!-- 以mysql为例，所以引入mysql的坐标，mysql要开启binlog-->
<!-- https://mvnrepository.com/artifact/io.debezium/debezium-connector-mysql -->
        <dependency>
            <groupId>io.debezium</groupId>
            <artifactId>debezium-connector-mysql</artifactId>
            <version>2.5.1.Final</version>
        </dependency>

三、启动DebeziumEngine

官网链接：Debezium Engine :: Debezium Documentation

mysql要开启binlog

        //定义任务ID
        String taskId = "123456";
        // 构建配置
        Properties props = new Properties();
        props.setProperty("name", "mysql-connector");
        props.setProperty("connector.class", "io.debezium.connector.mysql.MySqlConnector");
        props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore");
        props.setProperty("offset.storage.file.filename", "E:\\debezium\\offset\\offsets.dat");
        props.setProperty("offset.flush.interval.ms", "10000");
        props.setProperty("database.history", "io.debezium.relational.history.FileDatabaseHistory");
        props.setProperty("database.history.file.filename", "E:\\debezium\\history\\history.dat");
        props.setProperty("snapshot.mode","initial");
        props.setProperty("database.hostname", "10.0.**.**");
        props.setProperty("database.port", "3306");
        props.setProperty("database.user", "root");
        props.setProperty("database.password", "root");
        //props.put("database.whitelist","test");
        props.setProperty("table.include.list","my_database.test");
        props.setProperty("database.server.id", "88886666");
        props.setProperty("database.server.name", "test_88886666");

        props.setProperty("topic.prefix", "aaa");
        props.setProperty("schema.history.internal", "io.debezium.storage.file.history.FileSchemaHistory");
        props.setProperty("schema.history.internal.file.filename", "E:\\debezium\\history\\schemahistory.dat");

        DebeziumEngine<ChangeEvent<String,String>> engine = DebeziumEngine.create(Json.class)
                .using(props)
                .using(new DebezuimCompletionCallback(taskId))
                .using(new DebeziumConnectorCallback())
                .notifying(new DebeziumChangeConsumer(taskId))
                .build();

        Executors.newSingleThreadExecutor().execute(engine);

四、扩展类

1、任务结束收到的通知

/**
 * 任务结束收到通知
 */
public class DebezuimCompletionCallback implements DebeziumEngine.CompletionCallback {

    private static final Logger logger = LoggerFactory.getLogger(DebezuimCompletionCallback.class);

    private String taskId;

    public DebezuimCompletionCallback(String taskId){
        this.taskId = taskId;
    }

    @Override
    public void handle(boolean b, String s, Throwable throwable) {
        if(!b){
            logger.error("Debezium【{}】 error:{}",taskId,s);
            logger.error("Debezium error info:",throwable);
            return;
        }
        logger.info("Debezium【{}】 completionCallback!",taskId);
    }
}

2、Debezium连接的回调

/**
 * Debezium连接的回调
 */
public class DebeziumConnectorCallback implements DebeziumEngine.ConnectorCallback {

    private static final Logger logger = LoggerFactory.getLogger(DebeziumConnectorCallback.class);

    @Override
    public void connectorStarted() {
        logger.info("Debezium connectorStarted!");
    }

    @Override
    public void connectorStopped() {
        logger.info("Debezium connectorStopped!");
    }

    @Override
    public void taskStarted() {
        logger.info("Debezium taskStarted!");
    }

    @Override
    public void taskStopped() {
        logger.info("Debezium taskStopped!");
    }
}

3、debezium数据变更入口

/**
 * Debezium数据变更入口
 */
public class DebeziumChangeConsumer implements DebeziumEngine.ChangeConsumer<ChangeEvent<String,String>> {

    private static final Logger logger = LoggerFactory.getLogger(DebeziumChangeConsumer.class);

    private String taskId;

    public DebeziumChangeConsumer(String taskId){
        this.taskId = taskId;
    }
    /**
     * debezium拿到数据的入口类
     * @param list
     * @param recordCommitter
     * @throws InterruptedException
     */
    @Override
    public void handleBatch(List<ChangeEvent<String, String>> list, DebeziumEngine.RecordCommitter<ChangeEvent<String, String>> recordCommitter) throws InterruptedException {
        ListIterator<ChangeEvent<String, String>> iterator = list.listIterator();
        //开始处理
        while (iterator.hasNext()){
            ChangeEvent<String, String> record = iterator.next();
            logger.debug("Debezium 解析到一条数据 处理前 Key='{}' value='{}'",record.key(),record.value());
            recordCommitter.markProcessed(record);
        }
        recordCommitter.markBatchFinished();
    }
}

五、DebeziumEngine的停止

可以在启动DebeziumEngine的时候，将DebeziumEngine对象保存到一个Map中，后续通过这个Map获取到对应的DebeziumEngine进行停止

debeziumEngine.close();

六、自定义一个Debezium框架级通知

官网：Debezium notifications :: Debezium Documentation

1、定义类MyNotificationChannel并实现NotificationChannel

import io.debezium.config.CommonConnectorConfig;
import io.debezium.pipeline.notification.Notification;
import io.debezium.pipeline.notification.channels.NotificationChannel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * debezium通知
 */
public class MyNotificationChannel implements NotificationChannel {

    private final Logger logger = LoggerFactory.getLogger(MyNotificationChannel .class);

    private CommonConnectorConfig commonConnectorConfig;
    @Override
    public void init(CommonConnectorConfig commonConnectorConfig) {
        this.commonConnectorConfig = commonConnectorConfig;
    }

    @Override
    public String name() {
        return "my_notice";
    }

    @Override
    public void send(Notification notification) {
        logger.info("收到Debezium消息:{}",notification.toJson());
    }

    @Override
    public void close() {
        logger.info("MyNotificationChannel close!");
    }
}

2、覆盖debezium默认配置文件

1）在resourcesc的META-INF下创建services包创建文件

2）创建io.debezium.pipeline.notification.channels.NotificationChannel文件，将刚才自己创建的类加进去

io.debezium.pipeline.notification.channels.SinkNotificationChannel
io.debezium.pipeline.notification.channels.LogNotificationChannel
io.debezium.pipeline.notification.channels.jmx.JmxNotificationChannel
com.test.handler.debezium.MyNotificationChannel

3、在debeziumEngine配置文件中指定通知

props.setProperty("notification.enabled.channels","my_notice");

七、各种连接器配置示例

1、postgres

Properties props = new Properties();
props.setProperty("slot.name", "debezium1");
props.setProperty("name", "postgres-connector");
props.setProperty("connector.class", "io.debezium.connector.postgresql.PostgresConnector");
props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore");
props.setProperty("offset.storage.file.filename", "E:\\debezium\\offset\\pg-offsets.dat");
props.setProperty("offset.flush.interval.ms", "1000");
props.setProperty("database.history", "io.debezium.relational.history.FileDatabaseHistory");
props.setProperty("database.history.file.filename", "E:\\debezium\\history\\pg-history.dat");
//解码器，支持的格式decoderbufs（需要安装插件）、 和pgoutput
props.setProperty("plugin.name","pgoutput");
props.setProperty("snapshot.mode","never");
props.setProperty("database.hostname", "10.0.*.*");
props.setProperty("database.port", "5432");
props.setProperty("database.user", "postgres");
props.setProperty("database.password", "***");
props.setProperty("database.dbname", "test");
props.setProperty("table.include.list","public.dict");
props.setProperty("database.server.id", "88887777");
props.setProperty("database.server.name", "test_88887777");

2、mongodb

Properties props = new Properties();
props.setProperty("name", "mongodb-connector");
props.setProperty("connector.class", "io.debezium.connector.mongodb.MongoDbConnector");
props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore");
props.setProperty("offset.storage.file.filename", "E:\\debezium\\offset\\mongodb_offsets.dat");
props.setProperty("offset.flush.interval.ms", "10000");
props.setProperty("snapshot.mode","initial_only");
props.setProperty("mongodb.connection.string", "mongodb://root:***@127.0.0.1:27017/?replicaSet=rs0");
//props.setProperty("mongodb.user", "root");
//props.setProperty("mongodb.password", "***");
//props.setProperty("database.include.list","test");
props.setProperty("collection.include.list","test.dict,test.users");
props.setProperty("topic.prefix", "aaa");
props.setProperty("converter.schemas.enable", "false");