Canal概述:
canal是由Alibaba开源的一个基于binlog的增量日志组件,其核心原理是canal伪装成Mysql的slave,发送dump协议获取binlog,解析并存储起来给客户端消费。
Canal的工作流程:
- canal模拟mysql slave的交互协议,伪装自己为mysql slave,向mysql master发送dump协议
- mysql master收到dump请求,开始推送binary log给slave(也就是canal)
- canal解析binary log对象(原始为byte流)
Canal下载地址: https://github.com/alibaba/canal/releases
Canal 单机模式使用流程:
1.开启Mysql数据库binlog,并设置为row格式
2.为数据库创建用户,并赋予REPLICATION SLAVE, REPLICATION CLIENT的权限,否则用这些用户连接canal时,无法读取binlog. (默认的root账户肯定也可以)
3.解压下载的canal.deployer-1.1.1.tar.gz到指定的任意目录
mkdir canal
tar -zxvf canal.deployer-1.1.1.tar.gz
4.进入解压目录进行配置修改
vi canal/conf/example/instance.properties
#################################################
## mysql serverId
canal.instance.mysql.slaveId = 1234
#position info,需要改成自己的数据库信息
canal.instance.master.address = 127.0.0.1:3306
canal.instance.master.journal.name =
canal.instance.master.position =
canal.instance.master.timestamp =
#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
#username/password,需要改成自己的数据库信息
canal.instance.dbUsername = canal
canal.instance.dbPassword = canal
canal.instance.defaultDatabaseName =
canal.instance.connectionCharset = UTF-8
#table regex
canal.instance.filter.regex = .\..
#################################################
5.启动canal
sh bin/startup.sh
6.查看日志是否正常启动
vi logs/canal/canal.log
具体instance的日志
vi logs/example/example.log
7.编写java client 程序
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>1.1.1</version>
</dependency>
</dependencies>
package com.mytest.canal;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import com.alibaba.otter.canal.protocol.CanalEntry.*;
import com.alibaba.otter.canal.protocol.Message;
import com.google.protobuf.InvalidProtocolBufferException;
import com.mytest.canal.bean.CanalStuMsgBean;
import com.mytest.canal.bean.CanalStuScoreBean;
import com.mytest.canal.bean.StudentMsgBean;
import com.mytest.canal.bean.StuentScoreBean;
import com.mytest.kafka.KafkaProduceUtils;
import com.mytest.utils.DateUtil;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
public class CanalClientTest {
private static KafkaProduceUtils kafkaProduceUtils;
public static StringBuilder builder = new StringBuilder();
public static void main(String args[]) {
// kafkaProduceUtils = new KafkaProduceUtils("my_test_topic");
// 创建链接
CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("127.0.0.1",
11111), "example", "", "");
int batchSize = 1000;
int emptyCount = 0;
try {
connector.connect();
connector.subscribe(".*\\..*");
connector.rollback();
int totalEmptyCount = 12000;
while (emptyCount < totalEmptyCount) {
Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
emptyCount++;
System.out.println("empty count : " + emptyCount);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
} else {
emptyCount = 0;
System.out.printf("message[batchId=%s,size=%s] \n", batchId, size);
printEntry(message.getEntries());
}
connector.ack(batchId); // 提交确认
// connector.rollback(batchId); // 处理失败, 回滚数据
}
System.out.println("empty too many times, exit");
} catch (Exception e) {
e.printStackTrace();
} finally {
connector.disconnect();
}
}
private static void printEntry(List<Entry> entrys) {
for (Entry entry : entrys) {
if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
continue;
}
RowChange rowChage = null;
try {
rowChage = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
e);
}
EventType eventType = rowChage.getEventType();
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(),
entry.getHeader().getSchemaName(), entry.getHeader().getTableName(),
eventType));
for (RowData rowData : rowChage.getRowDatasList()) {
if (eventType == EventType.DELETE) {
printColumn(rowData.getBeforeColumnsList());
} else if (eventType == EventType.INSERT) {
printColumn(rowData.getAfterColumnsList());
} else {
System.out.println("-------> before");
printColumn(rowData.getBeforeColumnsList());
System.out.println("-------> after");
printColumn(rowData.getAfterColumnsList());
}
}
}
}
private static void printColumn(List<Column> columns) {
for (Column column : columns) {
System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated());
}
}
}
8.运行上述java程序
9.对mysql数据库进行操作,建表,插入数据,观察java client端是否有对应数据库日志打印出来
10.关闭java client, 关闭 canal server
sh bin/stop.sh
本文参考地址为canal github地址: https://github.com/alibaba/canal/wiki