ogg 源端到目标端到kafka（数据库mysql）

OGG安装

#groupadd oinstall
#useradd -G oinstall oracle

mkdir -p /u01/ogg
cp ggs_Linux_x64_MySQL_64bit.tar /u01/ogg
chown -R oracle:oinstall /u01
#su - oracle
cd /u01/ogg
tar xvf ggs_Linux_x64_MySQL_64bit.tar

–配置参数
vim ~/.bash_profile

export OGG_HOME=/u01/ogg
export PATH=$OG{G}_{H}OME:$PATH

export
LD_LIBRARY_PATH=$ORACL{E}_{H}OME/lib:/lib:/usr/lib:$ORACLE_HOME/jdk/jre/lib/amd64:$ORACL{E}_{H}OME/jdk/jre/lib/amd64/server:$ORACLE_HOME/jdk/jre/lib/amd64/libjsig.so:$ORACL{E}_{H}OME/jdk/jre/lib/amd64/server/libjvm.so:$OGG_HOME/lib:$L{D}_{L}IBRAR{Y}_{P}ATHexportL{D}_{L}IBRAR{Y}_{P}ATH=$OGG_HOME:$LD_LIBRARY_PATH
source ~/.bash_profile

cd /u01/ogg
./ggsci

create subdirs

usermod -a -G mysql oracle

数据库配置

CREATE USER ogg@% IDENTIFIED WITH mysql_native_password BY ‘welcome1’ PASSWORD
EXPIRE NEVER;
GRANT ALL PRIVILEGES ON test_db.* TO ‘ogg’@‘%’;

edit param mgr

------配置mgr -----

PORT 7809
DYNAMICPORTLIST 7810-7909
PURGEOLDEXTRACTS ./dirdat/*,usecheckpoints, minkeepdays 3
LAGREPORTHOURS 1
LAGINFOMINUTES 30
LAGCRITICALMINUTES 45

------配置mgr end----

dblogin sourcedb test_db@127.0.0.1:3306 userid ogg password welcome1

add checkpointtable test_db.ogg_checkpointtab

edit param extbill

------源端配置抽取进程 begin—

extract extbill
sourcedb test_db@127.0.0.1:3306 userid ogg password welcome1
exttrail /u01/ogg/dirdat/bi
discardfile /u01/ogg/dirrpt/extbill.dsc,append
TranLogOptions AltLogDest /data/mysql/binlogs/binlog.index

GETUPDATEBEFORES
NOCOMPRESSUPDATES
NOCOMPRESSDELETES

table test_db.ogg_test;

add extract extbill,tranlog,begin now
add exttrail /u01/ogg/dirdat/bi,extract extbill

------配置抽取进程 end—

------源端配置pump进程 begin------

GSCI (dciuap14) 10> edit param pumpbill

extract pumpbill
passthru
sourcedb test_db@127.0.0.1:3306 userid ogg password welcome1
rmthost zt_kafka mgrport 7809
rmttrail /u01/ogg/dirdat/bi
GETUPDATEBEFORES
dynamicresolution
numfiles 3000
table test_db.po_ogg_test;

add extract pumpbill,exttrailsource /u01/ogg/dirdat/bi
add rmttrail /u01/ogg/dirdat/bi,extract pumpbill ,MEGABYTES 500

-----配置pump进程end------

GGSCI (dciuap14) 13> edit param ogg_bi

-------配置define文件----defsfile /u01/ogg/dirdef/ogg_bi
sourcedb test_db@127.0.0.1:3306 userid ogg password welcome1
table test_db.ogg_test;

#OGG主目录下执行
./defgen paramfile dirprm/ogg_bi.prm

#将生成的/u01/ogg/dirdef/ogg_fu发送的目标端ogg目录下的dirdef里,等到目标端安装后在复制
#oracle 用户密码welcome1
scp -r /u01/ogg/dirdef/ogg_bi oracle@zt_kafka:/u01/ogg/dirdef/

3、目标端配置

edit mgr

PORT 7809
DYNAMICPORTLIST 7810-7909
PURGEOLDEXTRACTS ./dirdat/*,usecheckpoints, minkeepdays 3
ACCESSRULE, PROG *, IPADDR 192.168.10.203, ALLOW

LAGREPORTHOURS 1
LAGINFOMINUTES 30
LAGCRITICALMINUTES 45

-------复制进程------
REPLICAT repbi
sourcedefs /u01/ogg/dirdef/ogg_bi
TARGETDB LIBFILE libggjava.so SET property=dirprm/kafka.props

HANDLECOLLISIONS
REPORTCOUNT EVERY 1 MINUTES, RATE

GROUPTRANSOPS 10000
GETUPDATEBEFORES

MAP test_db.ogg_test,target test_db.ogg_test;

添加trail文件到replicate进程

add replicat repmp exttrail /u01/ogg/dirdata/bi,checkpointtable
test_db.ogg_checkpointtab

启动服务

源端： start mgr 先启动管理

         start  extbill
         start pumpbill

目标端： start mgr 先启动管理

        start  repbi

测试

#目标端，连接到kafka
–查看topic
/u01/kafka_2.13-2.5.0/bin/kafka-topics.sh --list --bootstrap-server test-ogg-01:9092,test-ogg-02:9092,test-ogg-03:9092
–跟踪
/u01/kafka_2.13-2.5.0/bin/kafka-console-consumer.sh --bootstrap-server test-ogg-01:9092,test-ogg-02:9092,test-ogg-03:9092 --topic
ogg_test --from-beginning