debezium 捕获mysql数据二（CDC）

debezium 捕获mysql数据二（CDC）

项目结构

引入依赖

        
<debezium.version>2.6.2.Final</debezium.version>

<dependency>
  <groupId>io.debezium</groupId>
  <artifactId>debezium-api</artifactId>
  <version>${debezium.version}</version>
</dependency>
<dependency>
  <groupId>io.debezium</groupId>
  <artifactId>debezium-embedded</artifactId>
  <version>${debezium.version}</version>
</dependency>

配置 MysqlDebeziumConfig：

@Configuration
@ConfigurationProperties(prefix = "dbzium.mysql")
@Data
public class MysqlDebeziumConfig {

    private String host;
    private String port;
    private String user;
    private String passwd;
    private String dbs;
    private String serverId;
}

监听数据变动

@Slf4j
public abstract class AbstractDebeziumDataListener implements ApplicationRunner, AutoCloseable {

    private DebeziumEngine<ChangeEvent<String, String>> debeziumEngine;

    private void init() {
        this.debeziumEngine = DebeziumEngine.
                create(Json.class)
                .using(getProperties())
                .notifying(this::handleEvent)
                .using(new DebeziumEngine.CompletionCallback() {
                    @Override
                    public void handle(boolean b, String s, Throwable throwable) {
                        if (throwable != null) {
                            log.error("{} init error", this.getClass().getSimpleName(), throwable);
                        } else {
                            log.info("{} init success", this.getClass().getSimpleName());
                        }
                    }
                })
                .using(new DebeziumEngine.ConnectorCallback() {
                    @Override
                    public void connectorStarted() {
                        DebeziumEngine.ConnectorCallback.super.connectorStarted();
                        log.info("{} connectorStarted", this.getClass().getSimpleName());
                    }

                    @Override
                    public void connectorStopped() {
                        DebeziumEngine.ConnectorCallback.super.connectorStopped();
                        log.info("{} connectorStopped", this.getClass().getSimpleName());
                    }

                    @Override
                    public void taskStarted() {
                        DebeziumEngine.ConnectorCallback.super.taskStarted();
                        log.info("{} taskStarted", this.getClass().getSimpleName());
                    }

                    @Override
                    public void taskStopped() {
                        DebeziumEngine.ConnectorCallback.super.taskStopped();
                        log.info("{} taskStopped", this.getClass().getSimpleName());
                    }
                }).build();
    }

    protected abstract void handleEvent(List<ChangeEvent<String, String>> changeEvents,
                                        DebeziumEngine.RecordCommitter<ChangeEvent<String, String>> changeEventRecordCommitter);


    protected abstract Executor getExecutor();

    /**
     * 配置信息
     *
     * @return
     */
    protected abstract Properties getProperties();

    @Override
    public void run(ApplicationArguments args) throws Exception {
        init();
        start0();
    }

    private void start0() {
        getExecutor().execute(debeziumEngine);
    }


    @Override
    public void close() throws Exception {
        debeziumEngine.close();
    }
}

mysql 实现:

@Component
@Slf4j
public class MysqlDebeziumDataListener extends AbstractDebeziumDataListener {

    @Autowired
    private MysqlDebeziumConfig config;

    @Override
    protected void handleEvent(List<ChangeEvent<String, String>> changeEvents, DebeziumEngine.RecordCommitter<ChangeEvent<String, String>> changeEventRecordCommitter) {
        Iterator<ChangeEvent<String, String>> iterator = changeEvents.iterator();
        while (iterator.hasNext()) {
            ChangeEvent<String, String> next = iterator.next();
            JSONObject jsonObject = JSON.parseObject(next.value());
            if (jsonObject==null || !jsonObject.containsKey("payload")){
                log.info("opt contains no payload: {}", JSON.toJSONString(JSON.parseObject(next.value()),SerializerFeature.PrettyFormat));
                return;
            }
            JSONObject payload = jsonObject.getJSONObject("payload");
            if (payload == null) {
                log.info("no opt record....");
                return;
            }
            String opt = payload.getString("op");
            JSONObject before = payload.getJSONObject("before");
            JSONObject after = payload.getJSONObject("after");
            log.info("opt: {}", opt);
            log.info("before: {}", before == null ? null : before.toJSONString());
            log.info("after: {}", after == null ? null : after.toJSONString());
             //提交!!!!
            try {
                changeEventRecordCommitter.markProcessed(next);
            } catch (InterruptedException e) {
                log.info("commit change event error: {}", JSON.toJSONString(next), e);
            }
        }
    }

    @Override
    protected Executor getExecutor() {
        return Executors.newSingleThreadExecutor(new ThreadFactory() {
            AtomicInteger cnt = new AtomicInteger(0);

            @Override
            public Thread newThread(Runnable r) {
                return new Thread(r, "mysql-dbzium" + cnt.getAndIncrement());
            }
        });
    }

    @Override
    protected Properties getProperties() {
          final Properties prop = new Properties();
        prop.setProperty("name", "engine");
        prop.setProperty("connector.class", "io.debezium.connector.mysql.MySqlConnector");
        prop.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore");
        prop.setProperty("offset.storage.file.filename", "F:\\abc\\dbzium\\offset1.dat");
        prop.setProperty("offset.flush.interval.ms", "60000");
        /* begin connector properties */
        prop.put("database.hostname", config.getHost());
        prop.put("database.port", config.getPort());
        prop.put("database.user", config.getUser());
        prop.put("database.password", config.getPasswd());
        //与mysql的server_id不一样
        prop.setProperty("database.server.id", config.getServerId());
        prop.setProperty("database.include.list", "test");//要捕获的数据库名
        prop.setProperty("table.include.list", "test.t_user");//要捕获的数据表
        prop.put("include.schema.changes", "false");
        prop.setProperty("topic.prefix", "my-app-connector");
        prop.setProperty("schema.history.internal", "io.debezium.storage.file.history.FileSchemaHistory");
        prop.setProperty("schema.history.internal.file.filename", "F:\\abc\\dbzium\\his1.dat");
        return prop;
    }


}

启动类:

@Slf4j
@SpringBootApplication
public class BootDebeziumApplication {

    public static void main(String[] args) {
        SpringApplication.run(BootDebeziumApplication.class, args);
    }

    @Autowired
    private List<AbstractDebeziumDataListener> listener;

    @PostConstruct
    public void init() {
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            for (AbstractDebeziumDataListener l : listener) {
                try {
                    log.info("AbstractDebeziumDataListener start close : {}", l.getClass().getSimpleName());
                    l.close();
                } catch (Exception e) {
                    log.info("AbstractDebeziumDataListener close error: {}", l.getClass().getSimpleName(), e);
                }
            }
        }));
    }
}

结果:

MysqlDebeziumDataListener  : opt: u
MysqlDebeziumDataListener  : before: {"name":"dsfsdf5","id":8}
MysqlDebeziumDataListener  : after: {"name":"tom","id":8}
MysqlDebeziumDataListener  : opt: d
MysqlDebeziumDataListener  : before: {"name":"ddd","id":6}
MysqlDebeziumDataListener  : after: null

注意

高版本创建 FileOffsetBackingStore 时，已经没有带参数的构造函数了，所以需要 重写KafkaConnectUtil

public class KafkaConnectUtil {

    public static final Converter converterForOffsetStore() {
        final JsonConverter converter = new JsonConverter();
        converter.configure(Collections.singletonMap(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, "false"), true);
        return converter;
    }

    public static final FileOffsetBackingStore fileOffsetBackingStore() {
        //改成无参就行
        return new FileOffsetBackingStore();
    }

    public static final MemoryOffsetBackingStore memoryOffsetBackingStore() {
        return new MemoryOffsetBackingStore();
    }

    public static final KafkaOffsetBackingStore kafkaOffsetBackingStore(Map<String, String> config) {
        final String clientId = "debezium-server";
        final Map<String, Object> adminProps = new HashMap<>(config);
        adminProps.put(CLIENT_ID_CONFIG, clientId + "shared-admin");

        Stream.of(
                DistributedConfig.BOOTSTRAP_SERVERS_CONFIG,
                DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG,
                DistributedConfig.OFFSET_STORAGE_PARTITIONS_CONFIG,
                DistributedConfig.OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG).forEach(prop -> {
            if (!adminProps.containsKey(prop)) {
                throw new DebeziumException(String.format(
                        "Cannot initialize Kafka offset storage, mandatory configuration option '%s' is missing",
                        prop));
            }
        });
        SharedTopicAdmin sharedAdmin = new SharedTopicAdmin(adminProps);
        return new KafkaOffsetBackingStore(sharedAdmin);
    }

}

mysql配置

$ cat conf/my.cnf 
[mysqld]
log_bin = mysql-bin
server_id = 1
binlog_format     = ROW                         #必须是ROW
binlog_row_image  = FULL                        #必须是
expire_logs_days  = 10                          #依据实际情况而定