linux 下载安装canal并运行示例

最近自己配置了下canal监控mysql的示例，和大家分享一下。

首先需要linux安装有mysql数据库，具体操作就不做说明了，我们需要修改mysql的my.cnf配置文件，在里面加入下面三行配置，开启mysql的binlog并配置模式为row：

[mysqld]

log-bin=mysql-bin #添加这一行就ok
binlog-format=ROW #选择row模式
server_id=1 #配置mysql replaction需要定义，不能和canal的slaveId重复

接下来我们需要为mysql创建用户并赋予权限，进入mysql后分别执行下面三行代码：

CREATE USER canal IDENTIFIED BY 'canal';  --创建canal用户
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';  --为canal用户赋予repication权限
FLUSH PRIVILEGES;     --修改完毕立即生效。

接下来我们需要下载canal

1、下载文件：wget https://github.com/alibaba/canal/releases/download/canal-1.1.0/canal.deployer-1.1.0.tar.gz

2、创建文件夹：mkdir /usr/local/canal

3、解压文件：tar -zxvf canal.deployer-1.1.0.tar.gz -C /usr/local/canal/

解压完成之后我们需要修改配置文件，

首先修改conf目录下的canal.properties这个文件：

canal.id=1234   #此处为canal服务的id,id随便，但不能和instance.properties这个配置文件的服务id相同
canal.ip=192.168.0.254 #此处为服务器的id地址，不要配置成127.0.0.1
canal.port=11111  #此处为配置的canal端口号

canal.metrics.pull.port=11112

canal.zkServers=
# flush data to zk
canal.zookeeper.flush.period = 1000
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
canal.file.flush.period = 1000

       canal.withoutNetty = false

       # tcp, kafka, rocketMQ

       canal.serverMode = tcp
## memory store RingBuffer size, should be Math.pow(2,n)
canal.instance.memory.buffer.size = 16384
## memory store RingBuffer used memory unit size , default 1kb
canal.instance.memory.buffer.memunit = 1024
## meory store gets mode used MEMSIZE or ITEMSIZE
canal.instance.memory.batch.mode = MEMSIZE

## detecing config
canal.instance.detecting.enable =true
#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
canal.instance.detecting.sql = select 1
canal.instance.detecting.interval.time = 3
canal.instance.detecting.retry.threshold = 3
canal.instance.detecting.heartbeatHaEnable = false

# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
canal.instance.transaction.size =  1024
# mysql fallback connected to new master should fallback times
canal.instance.fallbackIntervalInSeconds = 60

修改example下的instance.properties配置文件：

################################################
## mysql serverId
canal.instance.mysql.slaveId =1   #此处是配置的canal伪装的mysql slaveid。不能和canal.properties中配置的canal服务id相同

# position info
canal.instance.master.address =127.0.0.1:3306 #此处配置自己需要监控的mysql的IP地址和端口号
canal.instance.master.journal.name =
canal.instance.master.position =
canal.instance.master.timestamp =

#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =

#canal.instance.tsdb.enable=true
#canal.instance.detecting.interval.time=3
#canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:}
#canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=123

#canal.instance.enableDruid=false

# username/password
canal.instance.dbUsername = canal #此处是我们为mysql配置的canal用户
canal.instance.dbPassword = 123 #此处是我们为mysql配置的canal用户的密码
canal.instance.defaultDatabaseName =test #此处配置我们需要监控的mysql的数据库，如不填写则监控全部数据库
canal.instance.connectionCharset = UTF-8 #此处配置canal的编码

# table regex
canal.instance.filter.regex =test\\..* #此处是监控的mysql数据库，如监控全部数据库可配置为 .*\\..*
# table black regex
canal.instance.filter.black.regex =

配置完上述信息之后可启动canal 并查看输出日志检查是否启动成功。

如报错检查修改的配置文件是否正确。具体报错下篇文章再进行详细说明并解决。

canal启动成功之后就可配置canal的客户端

需要在pom.xml中添加canal的jar包

<dependency>
   <groupId>com.alibaba.otter</groupId>
   <artifactId>canal.client</artifactId>
   <version>1.0.12</version>
</dependency>

更新依赖 mvn install

 

最后是CanalClient.java类代码

package com.carinet.bitfinance.util;

import java.net.InetSocketAddress;
import java.util.List;

import com.alibaba.fastjson.JSONObject;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
import com.alibaba.otter.canal.client.*;

/**
 * @ClassName CanalClient
 * @Description TODO
 * @Author duyuanyuan
 * @Date 2019/2/2115:30
 * @Version 1.0
 **/
public class CanalClient {
    public static void main(String args[]) {
        // 创建链接            instanceA
        CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("192.168.0.234",
                11111), "example", "", "");
        int batchSize = 1000;
        int emptyCount = 0;
        try {
            connector.connect();
            connector.subscribe(".*\\..*");
            connector.rollback();
            int totalEntryCount = 1200;
            while (emptyCount < totalEntryCount) {
                Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
                long batchId = message.getId();
                int size = message.getEntries().size();
                if (batchId == -1 || size == 0) {
                    emptyCount++;
                    System.out.println("empty count : " + emptyCount);
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                } else {
                    emptyCount = 0;
                    printEntry(message.getEntries());
                }
                connector.ack(batchId); // 提交确认
            }
            System.out.println("empty too many times, exit");
        }catch (Exception e){
            //connector.rollback(batchId); // 处理失败, 回滚数据
        }
        finally {
            connector.disconnect();
        }
    }

    private static void printEntry( List<Entry> entrys) {
        for (Entry entry : entrys) {
            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                continue;
            }
            RowChange rowChage = null;
            try {
                rowChage = RowChange.parseFrom(entry.getStoreValue());
            } catch (Exception e) {
                throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
            }

            EventType eventType = rowChage.getEventType();
            System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
                    entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(),
                    entry.getHeader().getSchemaName(), entry.getHeader().getTableName(),
                    eventType));
            for (RowData rowData : rowChage.getRowDatasList()) {
                if (eventType == EventType.DELETE) {
                    printColumn(rowData.getBeforeColumnsList());
                    //此处可以将监控到的rowData.getAfterColumnsList()集合数据更新或者同步到redis或者es中
                } else if (eventType == EventType.INSERT) {
                    printColumn(rowData.getAfterColumnsList());
                     //此处可以将监控到的rowData.getAfterColumnsList()集合数据更新或者同步到redis或者es中
                } else {
                    System.out.println("-------> before");
                    printColumn(rowData.getBeforeColumnsList());
                    System.out.println("-------> after");
                    printColumn(rowData.getAfterColumnsList());
                     //此处可以将监控到的rowData.getAfterColumnsList()集合数据更新或者同步到redis或者es中
                }
            }
        }
    }

    private static void printColumn( List<Column> columns) {
        for (Column column : columns) {
            System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
        }
    }

}