debezium 捕获mysql数据（CDC）

debezium 捕获mysql数据（CDC）

引入依赖

        
<debezium.version>1.3.1.Final</debezium.version>

<dependency>
  <groupId>io.debezium</groupId>
  <artifactId>debezium-api</artifactId>
  <version>${debezium.version}</version>
</dependency>
<dependency>
  <groupId>io.debezium</groupId>
  <artifactId>debezium-embedded</artifactId>
  <version>${debezium.version}</version>
  <exclusions>
      <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-log4j12</artifactId>
      </exclusion>
      <exclusion>
          <groupId>org.reflections</groupId>
          <artifactId>reflections</artifactId>
      </exclusion>
  </exclusions>
</dependency>

配置 MysqlDebeziumConfig：

@Configuration
@ConfigurationProperties(prefix = "dbzium.mysql")
@Data
public class MysqlDebeziumConfig {

    private String host;
    private String port;
    private String user;
    private String passwd;
    private String dbs;

}

监听数据变动

@Slf4j
public abstract class AbstractDebeziumDataListener implements ApplicationRunner, AutoCloseable {

    private DebeziumEngine<ChangeEvent<String, String>> debeziumEngine;

    private void init() {
        this.debeziumEngine = DebeziumEngine.
                create(Json.class)
                .using(getProperties())
                .notifying(this::handleEvent)
                .using(new DebeziumEngine.CompletionCallback() {
                    @Override
                    public void handle(boolean b, String s, Throwable throwable) {
                        if (throwable != null) {
                            log.error("{} init error", this.getClass().getSimpleName(), throwable);
                        } else {
                            log.info("{} init success", this.getClass().getSimpleName());
                        }
                    }
                })
                .using(new DebeziumEngine.ConnectorCallback() {
                    @Override
                    public void connectorStarted() {
                        DebeziumEngine.ConnectorCallback.super.connectorStarted();
                        log.info("{} connectorStarted", this.getClass().getSimpleName());
                    }

                    @Override
                    public void connectorStopped() {
                        DebeziumEngine.ConnectorCallback.super.connectorStopped();
                        log.info("{} connectorStopped", this.getClass().getSimpleName());
                    }

                    @Override
                    public void taskStarted() {
                        DebeziumEngine.ConnectorCallback.super.taskStarted();
                        log.info("{} taskStarted", this.getClass().getSimpleName());
                    }

                    @Override
                    public void taskStopped() {
                        DebeziumEngine.ConnectorCallback.super.taskStopped();
                        log.info("{} taskStopped", this.getClass().getSimpleName());
                    }
                }).build();
    }

    protected abstract void handleEvent(List<ChangeEvent<String, String>> changeEvents,
                                        DebeziumEngine.RecordCommitter<ChangeEvent<String, String>> changeEventRecordCommitter);


    protected abstract Executor getExecutor();

    /**
     * 配置信息
     *
     * @return
     */
    protected abstract Properties getProperties();

    @Override
    public void run(ApplicationArguments args) throws Exception {
        init();
        start0();
    }

    private void start0() {
        getExecutor().execute(debeziumEngine);
    }


    @Override
    public void close() throws Exception {
        debeziumEngine.close();
    }
}

mysql 实现:

@Component
@Slf4j
public class MysqlDebeziumDataListener extends AbstractDebeziumDataListener {

    @Autowired
    private MysqlDebeziumConfig config;

    @Override
    protected void handleEvent(List<ChangeEvent<String, String>> changeEvents, DebeziumEngine.RecordCommitter<ChangeEvent<String, String>> changeEventRecordCommitter) {
        Iterator<ChangeEvent<String, String>> iterator = changeEvents.iterator();
        while (iterator.hasNext()) {
            ChangeEvent<String, String> next = iterator.next();
            JSONObject jsonObject = JSON.parseObject(next.value());
            if (jsonObject==null || !jsonObject.containsKey("payload")){
                log.info("opt contains no payload: {}", JSON.toJSONString(JSON.parseObject(next.value()),SerializerFeature.PrettyFormat));
                return;
            }
            JSONObject payload = jsonObject.getJSONObject("payload");
            if (payload == null) {
                log.info("no opt record....");
                return;
            }
            String opt = payload.getString("op");
            JSONObject before = payload.getJSONObject("before");
            JSONObject after = payload.getJSONObject("after");
            log.info("opt: {}", opt);
            log.info("before: {}", before == null ? null : before.toJSONString());
            log.info("after: {}", after == null ? null : after.toJSONString());
        }
    }

    @Override
    protected Executor getExecutor() {
        return Executors.newSingleThreadExecutor(new ThreadFactory() {
            AtomicInteger cnt = new AtomicInteger(0);

            @Override
            public Thread newThread(Runnable r) {
                return new Thread(r, "mysql-dbzium" + cnt.getAndIncrement());
            }
        });
    }

    @Override
    protected Properties getProperties() {
        Properties prop = new Properties();
        prop.put("name", "customer-mysql-connector");
        prop.put("connector.class", "io.debezium.connector.mysql.MySqlConnector");
        prop.put("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore");

        //偏移量文件
        prop.put("offset.storage.file.filename", "F:\\abc\\dbzium\\offset.dat");
        // 尝试提交偏移量的时间间隔。默认值为 1分钟
        prop.put("offset.flush.interval.ms", "0");
        prop.put("database.hostname", config.getHost());
        prop.put("database.port", config.getPort());
        prop.put("database.user", config.getUser());
        prop.put("database.password", config.getPasswd());
        // 要从中捕获更改的数据库的名称
       /* prop.put("database.dbname", "test");
        // Debezium 应捕获其更改的所有表的列表
        prop.put("database.include.list", "test.t_user");*/
        prop.setProperty("database.include.list", "test");//要捕获的数据库名
        prop.setProperty("table.include.list", "test.t_user");//要捕获的数据表
        prop.put("include.schema.changes", "false");
        prop.put("database.server.id", "1");
        prop.put("database.server.name", "c_mysql_connector");
        prop.put("database.history", "io.debezium.relational.history.FileDatabaseHistory");
        //读取历史记录文件
        prop.put("database.history.file.filename", "F:\\abc\\dbzium\\his.dat");
        return prop;
    }


}

启动类:

@Slf4j
@SpringBootApplication
public class BootDebeziumApplication {

    public static void main(String[] args) {
        SpringApplication.run(BootDebeziumApplication.class, args);
    }

    @Autowired
    private List<AbstractDebeziumDataListener> listener;

    @PostConstruct
    public void init() {
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            for (AbstractDebeziumDataListener l : listener) {
                try {
                    log.info("AbstractDebeziumDataListener start close : {}", l.getClass().getSimpleName());
                    l.close();
                } catch (Exception e) {
                    log.info("AbstractDebeziumDataListener close error: {}", l.getClass().getSimpleName(), e);
                }
            }
        }));
    }
}

结果:

MysqlDebeziumDataListener  : opt: u
MysqlDebeziumDataListener  : before: {"name":"dsfsdf5","id":8}
MysqlDebeziumDataListener  : after: {"name":"tom","id":8}
MysqlDebeziumDataListener  : opt: d
MysqlDebeziumDataListener  : before: {"name":"ddd","id":6}
MysqlDebeziumDataListener  : after: null