mysql binlog 实时_实时抽取mysql的binlog日志到kafka的详细配置方法

完美使用canal实时抽取binlog日志到kafka的详细配置

# 第一步 数据库my.ini参数 设置binlog_format=ROW

# 第二步 数据库赋权

CREATE USER canal IDENTIFIED BY 'canal';

GRANT SELECT, SHOW VIEW, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';

FLUSH PRIVILEGES;

select version()

show grants for 'canal' ;

show binary logs

show variables like 'binlog_format';

show variables like '%server_id%';

show variables like 'log_bin';

SHOW MASTER STATUS

show SLAVE STATUS

# 第三步 canal server里配置源数据库 目标库 MQ等地址

https://github.com/alibaba/canal/wiki/Canal-Kafka-RocketMQ-QuickStart

###################################

node1-main/

canal.serverMode = kafka

canal.id = 1

canal.ip =

canal.port = 11111

canal.metrics.pull.port = 11112

canal.mq.topic=canal_main

canal.instance.master.address=172.21.xx.xx:3306

canal.instance.filter.regex=fj_prod_main\\.class_pub,fj_prod_main\\.class_pub_student,fj_prod_main\\.class_pub_task,fj_prod_main\\.parent_children,fj_prod_main\\.school,fj_prod_main\\.user_info_basic,fj_prod_main\\.user_info_expand,fj_prod_main\\.user_info_ability,fj_prod_main\\.user_sign_author,fj_prod_main\\.user_vip_valid_time

node2-data/

canal.serverMode = kafka

canal.id = 2

canal.ip =

canal.port = 11121

canal.metrics.pull.port = 11122

canal.mq.topic=canal_data

canal.instance.master.address=172.21.xx.xx:3306

canal.instance.filter.regex=fj_prod_data\\.behavior_read_history,fj_prod_data\\.behavior_answer_record,fj_prod_data\\.essay_appreciate,fj_prod_data\\.level,fj_prod_data\\.summary,fj_prod_data\\.summary_collect,fj_prod_data\\.summary_essay,fj_prod_data\\.summary_mark,fj_prod_data\\.summary_share,fj_prod_data\\.topic

node3-stat/

canal.serverMode = kafka

canal.id = 3

canal.ip =

canal.port = 11131

canal.metrics.pull.port = 11132

canal.mq.topic=canal_stat

canal.instance.master.address=172.21.xx.xx:3306

canal.instance.filter.regex=fj_prod_stats\\.eval_history

解压并且稍微修改即可使用。