解决Debezium搭配OpenLogReplicator报scn为null无法采集变更数据的问题

问题描述

debezium捕捉Oracle行级变更数据目前支持三种方式：LogMiner、XStream API 和 OpenLogReplicator（olr），默认采用的是Oracle提供的LogMiner。由于特殊原因我们尝试使用OpenLogReplicator，在抽取数据过程中遇到了NPE问题：

"io.debezium.connector.oracle.olr.client.StreamingEvent.getScn()" is null

OpenLogReplicator的运行仅需要1个配置文件OpenLogReplicator.json，官方给的一个参数比较完整的样例为：

{
  "version": "1.5.0",
  "dump-redo-log": 0,
  "dump-raw-data": 0,
  "dump-path": "/opt/dump",
  "log-level": 3,
  "trace": 0,
  "source": [
    {
      "alias": "S1",
      "name": "O112A",
      "reader": {
        "type": "online",
        "path-mapping": ["/db/fra", "/opt/fast-recovery-area"],
        "redo-copy-path": "copy",
        "user": "user1",
        "password": "Password1",
        "server": "//host:1521/SERVICE",
        "host-timezone": "+00:00",
        "log-timezone": "+00:00",
        "db-timezone": "+00:00",
        "disable-checks": 0
      },
      "metrics": {
        "type": "prometheus",
        "bind": "127.0.0.1:8080",
        "tag-names": "all"
      },
      "format": {
        "type": "json",
        "db": 0,
        "interval-dts": 0,
        "interval-ytm": 0,
        "attributes": 0,
        "message": 0,
        "xid": 0,
        "timestamp": 0,
        "timestamp-tz": 0,
        "timestamp-all": 0,
        "char": 0,
        "scn": 0,
        "scn-all": 0,
        "rid": 0,
        "unknown": 0,
        "schema": 0,
        "column": 0,
        "unknown-type": 0,
        "flush-buffer": 1048576
      },
      "state": {
        "type": "disk",
        "path": "checkpoint",
        "interval-s": 600,
        "interval-mb": 100,
        "keep-checkpoints": 100,
        "schema-force-interval": 20
      },
      "debug": {
        "owner": "SYSTEM",
        "table": "OPENLOGREPLICATOR",
        "stop-log-switches": 0,
        "stop-checkpoints": 0,
        "stop-transactions": 0,
        "stop-dmls": 0
      },
      "arch": "online",
      "flags": 0,
      "memory": {
        "min-mb": 64,
        "max-mb": 1024,
        "read-buffer-max-mb": 256
      },
      "redo-read-sleep-us": 250000,
      "arch-read-sleep-us": 10000000,
      "arch-read-tries": 10,
      "redo-verify-delay-us": 250000,
      "refresh-interval-us": 10000000,
      "transaction-max-mb": 1000,
      "filter": {
        "table": [
          {"owner": "OWNER1", "table": "TABLENAME1", "key": "col1, col2, col3"},
          {"owner": "OWNER1", "table": "TABLENAME2", "condition": "!([op] == 'd' && [login username] == 'USER1')"},
          {"owner": "OWNER2", "table": "TAB.*"}
        ],
        "skip-xid": ["1234.ABC.12345678", "1234.0ABC.12345678", "09000D001D050000"],
        "dump-xid": ["1234.ABC.12345679", "1234.0ABC.12345679", "09000D001D050001"]
      }
    }
  ],
  "target": [
    {
      "alias": "K1",
      "source": "S1",
      "writer": {
        "type": "kafka",
        "topic": "O112A",
        "max-message-mb": 500,
        "properties": {
          "bootstrap.servers": "localhost:9092",
          "enable.idempotence": "false",
          "queue.buffering.max.messages": "200000"
        },
        "poll-interval-us": 100000,
        "queue-size": 65536
      }
    }
  ]
}

问题解决

查看OpenLogReplicator关于参数的说明，注意到有个参数：scn-all，如下图：

scn-all用于控制是否在每个消息中都带上scn字段，将其设置为1，然后重启olr进程。
发现不再报scn为null的错误，问题解决。