ogg mysql安装包_MySQL + Oracle GoldenGate + OGG Application Adpater

具体配置步骤如下：

一.配置Oracle GoldenGate For MySQL

1.创建用户

useradd oggusr

echo oracle | passwd --stdin oggusr

2.解压软件包

su - oggusr

unzip /opt/freeware/ggs_Linux_x64_MySQL_64bit.zip

mkdir ogg && cd ogg && tar xvf ../ggs_Linux_x64_MySQL_64bit.tar

3.运行ggsci

./ggsci

4.在ggsci中创建子目录

GGSCI (hadoop1) 1> CREATE SUBDIRS

5.退出ggsci

GGSCI (hadoop1) 1> exit

6.配置ogg for mysql前的准备

6.1 配置extract参数指定二进制索引文件的路径

TRANLOGOPTIONS ALTLOGDEST "/home/mysusr01/mysql/mylog/mysql-bin-hadoop1-39301.index"

7.配置源端

7.1 配置manager

GGSCI (hadoop1) 1> dblogin sourcedb etlanalydb1@CIT-EDC-ETLAPP1:39316, userid ggsuser,password ggsuser

GGSCI (hadoop1) 2> edit param mgr

port 7809

7.2 配置extract

GGSCI (hadoop1) 1> edit param ext1

extract ext1

setenv (MYSQL_HOME="/home/mysusr01/mysql")

TRANLOGOPTIONS ALTLOGDEST "/home/mysusr01/mysql/mylog/mysql-bin-hadoop1-39301.index"

sourcedb test@hadoop1:39301,userid root,password oracle

exttrail ./dirdat/e1

dynamicresolution

gettruncates

table test.*;

GGSCI (hadoop1) 17> add extract ext1,tranlog,begin now

GGSCI (hadoop1) 18> add exttrail ./dirdat/e1,extract ext1 ,megabytes 100

7.3 配置pump

GGSCI (hadoop1) 21> edit param pump1

extract pump1

rmthost localhost,mgrport 8809

rmttrail .dirdat/p1

passthru

gettruncates

table test.*;

GGSCI (hadoop1) 25> add extract pump1,exttrailsource ./dirdat/e1

GGSCI (hadoop1) 26> add rmttrail ./dirdat/p1,extract pump1,MEGABYTES 100

7.4 生成srcdef文件

--编辑参数文件

vi flatfile.prm

DEFSFILE ./dirdef/srcdef.def

sourcedb test@hadoop1:39301,userid root,password oracle

TABLE test.*;

--生成srcdef

./defgen paramfile dirprm/flatfile.prm

7.5启动进程

GGSCI>start mgr

GGSCI>start ext1

GGSCI>start pump1

GGSCI>info all

二.配置OGG Application Adapter

1.配置mgr

GGSCI>edit param mgr

port

7809

2.配置ffwriter属性

cp AdapterExamples/filewriter/ffwriter.properties

dirprm/

vim dirprm/ffwriter.properties

#------------------------

#LOGGING OPTIONS

#------------------------

goldengate.log.logname=ffwriter

goldengate.log.level=INFO

goldengate.log.modules=LOGMALLOC

goldengate.log.level.LOGMALLOC=ERROR

goldengate.log.tostdout=false

goldengate.log.tofile=true

#------------------------

#FLAT FILE WRITER OPTIONS

#------------------------

goldengate.flatfilewriter.writers=dsvwriter

goldengate.userexit.chkptprefix=ffwriter_

#------------------------

# dsvwriter options

#------------------------

dsvwriter.mode=DSV

dsvwriter.rawchars=false

dsvwriter.includebefores=false

#dsvwriter.includecolnames=false

dsvwriter.includecolnames=true

dsvwriter.omitvalues=false

dsvwriter.diffsonly=false

dsvwriter.omitplaceholders=false

#dsvwriter.files.onepertable=false

dsvwriter.files.prefix=csv

dsvwriter.files.data.rootdir=./dirout

dsvwriter.files.data.ext=_data.dsv

dsvwriter.files.data.tmpext=_data.dsv.temp

dsvwriter.files.data.rollover.time=10

#dsvwriter.files.data.rollover.size=

dsvwriter.files.data.norecords.timeout=10

dsvwriter.files.control.use=true

dsvwriter.files.control.ext=_data.control

dsvwriter.files.control.rootdir=./dirout

dsvwriter.dsv.nullindicator.chars=

dsvwriter.dsv.fielddelim.chars=|

dsvwriter.dsv.linedelim.chars=\n

dsvwriter.dsv.quotes.chars="

dsvwriter.dsv.quotes.escaped.chars=""

#dsvwriter.metacols=position,txind,opcode,timestamp,schema,table

dsvwriter.metacols=txind,opcode,timestamp,schema,table

dsvwriter.metacols.txind.fixedlen=1

dsvwriter.metacols.txind.begin.chars=B

dsvwriter.metacols.txind.middle.chars=M

dsvwriter.metacols.txind.end.chars=E

dsvwriter.files.formatstring=pump_%s_%t_%d_%05n

#------------------------

# ldvwriter options

#------------------------

ldvwriter.mode=LDV

ldvwriter.rawchars=true

ldvwriter.includebefores=false

ldvwriter.includecolnames=false

ldvwriter.files.onepertable=false

ldvwriter.files.data.rootdir=./dirout

ldvwriter.files.data.ext=.data

ldvwriter.files.data.tmpext=.temp

ldvwriter.files.data.rollover.time=10

ldvwriter.files.data.norecords.timeout=10

ldvwriter.files.control.use=true

ldvwriter.files.control.ext=.ctrl

ldvwriter.files.control.rootdir=./dirout

ldvwriter.metacols=position,timestamp,@TOKEN-RBA,@TOKEN-POS,opcode,txind,schema,table

ldvwriter.metacols.TOKEN-RBA.fixedlen=10

ldvwriter.metacols.TOKEN-POS.fixedlen=10

ldvwriter.metacols.timestamp.fixedlen=26

ldvwriter.metacols.schema.fixedjustify=right

ldvwriter.metacols.schema.fixedpadchar.chars=Y

ldvwriter.metacols.opcode.fixedlen=1

ldvwriter.metacols.opcode.insert.chars=I

ldvwriter.metacols.opcode.update.chars=U

ldvwriter.metacols.opcode.delete.chars=D

ldvwriter.metacols.txind.fixedlen=1

ldvwriter.metacols.txind.begin.chars=B

ldvwriter.metacols.txind.middle.chars=M

ldvwriter.metacols.txind.end.chars=E

ldvwriter.metacols.txind.whole.chars=W

ldvwriter.ldv.vals.missing.chars=M

ldvwriter.ldv.vals.present.chars=P

ldvwriter.ldv.vals.null.chars=N

ldvwriter.ldv.lengths.record.mode=binary

ldvwriter.ldv.lengths.record.length=4

ldvwriter.ldv.lengths.field.mode=binary

ldvwriter.ldv.lengths.field.length=2

ldvwriter.files.rolloveronshutdown=false

ldvwriter.statistics.toreportfile=false

ldvwriter.statistics.period=onrollover

ldvwriter.statistics.tosummaryfile=true

ldvwriter.statistics.overall=true

ldvwriter.statistics.summary.fileformat=schema,table,schemaandtable,total,gctimestamp,ctimestamp

ldvwriter.statistics.summary.delimiter.chars=|

ldvwriter.statistics.summary.eol.chars=\n

ldvwriter.metacols.position.format=dec

ldvwriter.writebuffer.size=36863

3.配置ffwriter进程

GGSCI>edidt param ffwriter

extract ffwriter

sourcedefs ./dirdef/srcdef.def

CUSEREXIT ./flatfilewriter.so CUSEREXIT

PASSTHRU INCLUDEUPDATEBEFORES,PARAMS "./dirprm/ffwriter.properties"

table test.*;

GGSCI>ADD EXTRACT ffwriter,

EXTTRAILSOURCE dirdat/p1

4.启动进程

GGSCI>start mgr

GGSCI>start ffwriter

注：以上配置都正常的话可以在dirout目录下生成对应的平面文件。

5.整个数据落地的架构图：