Goldengate同步oracle数据到kafka集群

		OGG版本
源端	oracle 192.168.0.10	Oracle GoldenGate Command Interpreter for Oracle Version 11.2.1.0.3
目标端	kafka 102.168.0.131	Oracle GoldenGate for Big Data Version 19.1.0.0.5 (Build 007)

源端

进入Oracle数据库

归档日志查看

archive log list

开启归档模式

SQL> shutdown immediate  #关闭数据库

SQL>  startup mount		#将数据库打开到mount状态

SQL> alter database archivelog		#开启归档

SQL> alter database open				#打开数据库到open正常模式

SQL> alter system archive log start; (启用自动归档)

mgr进程

PORT 7809
DYNAMICPORTLIST 7810-7860
AUTORESTART ER *, RETRIES 3, WAITMINUTES 5
PURGEOLDEXTRACTS ./dirdat/*, USECHECKPOINTS, MINKEEPDAYS 30
lagreporthours 1
laginfominutes 30
lagcriticalminutes 60

添加表附加日志

dblogin userid goldengate password goldengate

#库所有表
add trandata pxgl.*

#库指定表
add trandata scott.kafka

#查看表附加日志
info trandata scott.kafka

extract抽取进程

extract o_eka
userid goldengate,password goldengate
setenv(NLS_LANG=AMERICAN_AMERICA.AL32UTF8)
setenv(ORACLE_SID="oradb")         #数据库sid
warnlongtrans 2h,checkinterval 30m
exttrail /data/goldengate/dirdat/to
dboptions allowunusedcolumn
dynamicresolution
fetchoptions nousesnapshot
ddl include mapped
ddloptions addtrandata,report        #支持DDL
notcpsourcetimer
NOCOMPRESSDELETES
NOCOMPRESSUPDATES
GETUPDATEBEFORES

table SAPSR3.*;tokens (TKN-ROWID = @GETENV ("RECORD", "rowid")) keycols (owner) ; #当表无主键是，生成伪主键

=======================简单的===================

extract o_eka
dynamicresolution
SETENV (ORACLE_SID = "orcl")
SETENV (NLS_LANG = "american_america.AL32UTF8")
userid goldengate,password goldengate
exttrail /data/goldengate/dirdat/to
table SAPSR3.*;

添加extract进程

add extract o_eka,tranlog,begin now

添加trail文件的定义与extract进程绑定

add exttrail /data/goldengate/dirdat/to,extract o_eka

extract投递进程

edit params o_pka


extract o_pka
rmthost 192.168.0.131 mgrport 7809
PASSTHRU
numfiles 5000
rmttrail /opt/module/ogg/dirdat/po
dynamicresolution
table  pxgl.*;

添加投递进程

add extract o_pka,exttrailsource /data/goldengate/dirdat/to

add rmttrail /opt/module/ogg/dirdat/to,extract o_pka,MEGABYTES 20

配置define文件

表结构定义文件

edit params 文件名

defsfile /data/goldengate/dirdef/defines.def
userid goldengate,password goldengate
table SAPSR3.*;

#在Ogg目录下
#生成表结构定义文件
./defgen paramfile dirprm/文件名.prm

scp -r /data/goldengate/dirdef/defines.def itwang@192.168.0.131:/opt/module/ogg/dirdef/

目标端

添加checkpoint表

在确保zookeeper集群和kafka正常的情况下做下面配置：
checkpoint即复制可追溯的一个偏移量记录，在全局配置里添加checkpoint表即可。
edit  param  ./GLOBALS
CHECKPOINTTABLE ogg.checkpoint

配置replicat进程

GGSCI (zookeeper) 5> edit params o_rka
REPLICAT o_rka
sourcedefs /opt/module/ogg/dirdef/defines.def
TARGETDB LIBFILE libggjava.so SET property=dirprm/kafka.props
REPORTCOUNT EVERY 1 MINUTES, RATE 
GROUPTRANSOPS 10000
MAP SAPSR3.*, TARGET SAPSR3.*;


add replicat o_rka,exttrail /opt/module/ogg/dirdat/to,checkpointtable ogg.checkpoint

配置kafka.props

vim custom_kafka_producer.properties

bootstrap.servers=192.168.131:9092
acks=1
reconnect.backoff.ms=1000
value.serializer=org.apache.kafka.common.serialization.ByteArraySerializer
key.serializer=org.apache.kafka.common.serialization.ByteArraySerializer
batch.size=102400
linger.ms=10000

vim kafka.props 

[root@zookeeper dirprm]# pwd
/kafka/ogg12/dirprm
[root@zookeeper dirprm]# vim kafka.props 
写入下面内容
gg.handlerlist = kafkahandler
gg.handler.kafkahandler.type=kafka
gg.handler.kafkahandler.KafkaProducerConfigFile=custom_kafka_producer.properties
#The following resolves the topic name using the short table name
gg.handler.kafkahandler.topicMappingTemplate=kafka
#The following selects the message key using the concatenated primary keys
#gg.handler.kafkahandler.keyMappingTemplate=${primaryKeys}
gg.handler.kafkahandler.format=json
gg.handler.kafkahandler.SchemaTopicName=scott
gg.handler.kafkahandler.BlockingSend =true
gg.handler.kafkahandler.includeTokens=false
gg.handler.kafkahandler.mode=op
gg.handler.kafkahandler.format.includePrimaryKeys=true

goldengate.userexit.writers=javawriter
javawriter.stats.display=TRUE
javawriter.stats.full=TRUE

gg.log=log4j
gg.log.level=INFO

gg.report.time=30sec

#Sample gg.classpath for Apache Kafka
gg.classpath=dirprm/:/kafka/kafka/libs/*:/kafka/ogg12/:/kafka/ogg12/lib/*
#Sample gg.classpath for HDP
#gg.classpath=/etc/kafka/conf:/usr/hdp/current/kafka-broker/libs/*

javawriter.bootoptions=-Xmx512m -Xms32m -Djava.class.path=ggjava/ggjava.jar

#############################简单的##################################


gg.handlerlist=kafkahandler 
gg.handler.kafkahandler.type=kafka
gg.handler.kafkahandler.KafkaProducerConfigFile=custom_kafka_producer.properties
gg.handler.kafkahandler.topicMappingTemplate=test_ogg01 
gg.handler.kafkahandler.format=json 
gg.handler.kafkahandler.mode=op  
gg.classpath=dirprm/:/opt/module/kafka_2.12-3.0.1/libs/*:/opt/module/ogg/:/opt/module/ogg/lib/*
javawriter.bootoptions=-Xmx512m -Xms32m -Djava.class.path=ggjava/ggjava.jar

启动进程，查看进程状态

仅供参考