搭建环境使用的版本如下：
mysql-installer-community-8.0.20.0.msi
jdk1.8.0_251
apache-zookeeper-3.6.2-bin
kafka_2.13-2.7.0
canal.deployer-1.1.5-SNAPSHOT
Zookeeper 配置
下载完成后解压结构目录如下：
复制一份conf/zoo_sample.cfg改为zoo.cfg
编辑配置文件，主要配置代码如下：
# 日志以及数据存储路径自己定义
dataDir=D:\kaifa\env\zk\data
dataLogDir=D:\kaifa\env\zk\log
# 端口号
clientPort=2181
双击bin/zkServer.cmd启动，启动成功截图如下：
Kafka 配置
下载解压后结构目录如下图：
config/server.properties，主要配置如下：
# 和hosts文件里面的ip一致,端口为kafka启动端口
advertised.listeners=PLAINTEXT://172.20.10.3:9092
# 非负整数，用于唯一标识broker
broker.id=0
# 端口
port=9092
# 处理网络请求的线程数量，一般默认配置就好
num.network.threads=2
# 处理磁盘的线程数量，一般默认配置就好
num.io.threads=8
# socket server 发送数据缓冲区大小
socket.send.buffer.bytes=5242880
# socket server 接受数据缓冲区大小
socket.receive.buffer.bytes=5242880
# soket server 可接受最大消息大小，防止oom
socket.request.max.bytes=104857600
# kafka存放消息的目录
log.dirs=D:\kaifa\env\kafka_2.13-2.7.0\logs
# 每个topic默认partition数量，根据消费者实际情况配置，配置过小会影响消费性能
num.partitions=2
# 日志保留时间
log.retention.hours=168
# 日志保留大小
log.segment.bytes=536870912
# 日志 segment file 大小. 超过这个大小创建新segment file
log.retention.check.interval.ms=60000
# 是否开启压缩
log.cleaner.enable=false
# Zookeeper host和port
zookeeper.connect=localhost:2181
# 连接zookeeper超时时间, 时间少了有时候启动报错
zookeeper.connection.timeout.ms=1000000
# 消息体的最大大小, 单位是字节, 需要和producer.properties保持一致, 此处配置默认1mb, kafka消息过大会报错, 建议设置大一点
message.max.bytes=12695150
# replicas每次获取数据的最大字节数, 需要和consumer.properties保持一致
replica.fetch.max.bytes=12695150
offsets.topic.replication.factor=1
config/consumer.properties，主要配置如下：
bootstrap.servers=localhost:9092
group.id=test-consumer-group
# 和上述配置保持一致
fetch.message.max.bytes=12695150
config/producer.properties，主要配置如下：
bootstrap.servers=localhost:9092
# 和上述配置保持一致
max.request.size=12695150
compression.type=none
启动kafka，命令如下：
kafka-server-start.bat …\config\server.properties
启动成功截图如下：
Canal 配置
解压Canal.deployer 文件目录如下：
conf/canal.properties 配置如下：
#################################################
######### common argument #############
#################################################
#canal.manager.jdbc.url=jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8
#canal.manager.jdbc.username=root
#canal.manager.jdbc.password=121212
canal.id = 1
canal.ip =
canal.port = 11111
canal.metrics.pull.port = 11112
#zookeeper配置集群逗号隔开
canal.zkServers = 127.0.0.1:2181
# flush data to zk
canal.zookeeper.flush.period = 1000
canal.withoutNetty = false
# tcp, kafka, RocketMQ
#默认为TCP，也就是你通过官方的example可以在终端查看数据，我们修改为kafka
canal.serverMode = kafka
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
canal.file.flush.period = 1000
## memory store RingBuffer size, should be Math.pow(2,n)
canal.instance.memory.buffer.size = 16384
## memory store RingBuffer used memory unit size , default 1kb
canal.instance.memory.buffer.memunit = 1024
## meory store gets mode used MEMSIZE or ITEMSIZE
canal.instance.memory.batch.mode = MEMSIZE
canal.instance.memory.rawEntry = true
## detecing config
canal.instance.detecting.enable = false
#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
canal.instance.detecting.sql = select 1
canal.instance.detecting.interval.time = 3
canal.instance.detecting.retry.threshold = 3
canal.instance.detecting.heartbeatHaEnable = false
# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
canal.instance.transaction.size = 1024
# mysql fallback connected to new master should fallback times
canal.instance.fallbackIntervalInSeconds = 60
# network config
canal.instance.network.receiveBufferSize = 16384
canal.instance.network.sendBufferSize = 16384
canal.instance.network.soTimeout = 30
# binlog filter config
canal.instance.filter.druid.ddl = true
canal.instance.filter.query.dcl = false
canal.instance.filter.query.dml = false
canal.instance.filter.query.ddl = false
canal.instance.filter.table.error = false
canal.instance.filter.rows = false
canal.instance.filter.transaction.entry = false
# binlog format/image check
canal.instance.binlog.format = ROW,STATEMENT,MIXED
canal.instance.binlog.image = FULL,MINIMAL,NOBLOB
# binlog ddl isolation
canal.instance.get.ddl.isolation = false
# parallel parser config
canal.instance.parser.parallel = true
## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()
#canal.instance.parser.parallelThreadSize = 16
## disruptor ringbuffer size, must be power of 2
canal.instance.parser.parallelBufferSize = 256
# table meta tsdb info
# 关于tsdb概念，建议看一下官方文档，大概意思是canal在获取DDL的时候有可能获取的是错误的，
# 那么为了解决这个问题，所有DDL都会直接解析，重新生成到meta里面，我们可以用MySQL去保存，默认是H2
#
# 由于没有解决canal.instance.filter.regex过滤生效问题,所以直接关闭了
canal.instance.tsdb.enable = false
canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
#canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
#保存meta的数据库地址和数据库名称，用户密码
canal.instance.tsdb.url = jdbc:mysql://127.0.0.1:3306/hse_db_test?useSSL=false
canal.instance.tsdb.dbUsername = root
canal.instance.tsdb.dbPassword = mysql@1234
# dump snapshot interval, default 24 hour
canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire = 360
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
#################################################
######### destinations #############
#################################################
#canal实例
canal.destinations = example
# conf root dir
canal.conf.dir = ../conf
# auto scan instance dir add/remove and start/stop instance
canal.auto.scan = true
canal.auto.scan.interval = 5
#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml
canal.instance.global.mode = spring
canal.instance.global.lazy = false
#canal.instance.global.manager.address = 127.0.0.1:1099
#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
##################################################
######### MQ #############
##################################################
#kafka地址
canal.mq.servers = 127.0.0.1:9092
canal.mq.retries = 0
canal.mq.batchSize = 16384
#此配置等同于kafka的message.max.bytes, 设置消息,消息发送过大会报错
canal.mq.maxRequestSize = 12695150
canal.mq.lingerMs = 100
canal.mq.bufferMemory = 33554432
canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout = 100
canal.mq.flatMessage = true
canal.mq.compressionType = none
canal.mq.acks = all
# use transaction for kafka flatMessage batch produce
canal.mq.transaction = false
#canal.mq.properties. =
conf/example/instance.properties 配置如下：
#################################################
## mysql serverId , v1.0.26+ will autoGen
# canal.instance.mysql.slaveId=0
# enable gtid use true/false
canal.instance.gtidon=false
# position info
canal.instance.master.address=127.0.0.1:3306
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
canal.instance.master.gtid=
# rds oss binlog
canal.instance.rds.accesskey=
canal.instance.rds.secretkey=
canal.instance.rds.instanceId=
# table meta tsdb info
canal.instance.tsdb.enable=false
canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/hse_db_test?useSSL=false
canal.instance.tsdb.dbUsername=root
canal.instance.tsdb.dbPassword=mysql@1234
#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
#canal.instance.standby.gtid=
# username/password
canal.instance.dbUsername=root
canal.instance.dbPassword=mysql@1234
canal.instance.defaultDatabaseName=hse_db_test
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
# table regex
canal.instance.filter.regex=hse_db_test1.hse_demo
# table black regex
canal.instance.filter.black.regex=
# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch
# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch
# mq config
canal.mq.topic=example
# dynamic topic route by schema or table regex
#canal.mq.dynamicTopic=mytest1.user,mytest2\\..*,.*\\..*
canal.mq.partition=0
# hash partition config
#canal.mq.partitionsNum=3
#canal.mq.partitionHash=test.table:id^name,.*\\..*
#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
#################################################
双击bin/startup.bat 启动, logs/canal/canal.log 下图：
表示启动成功
相关报错信息可以在配置logs/example(topic名称)/example.log 查看
客户端接入
创建一个Spring boot 项目，pom.xml 部分配置如下：
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>0.10.0.1</version>
</dependency>
消费者工具类实例代码如下：
import java.sql.*;
import java.util.Arrays;
import java.util.Properties;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
public class KafkaConsumerTest implements Runnable {
static {
LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
Logger root = loggerContext.getLogger("root");
root.setLevel(Level.ERROR);
}
private final KafkaConsumer<String, String> consumer;
private ConsumerRecords<String, String> msgList;
private final String topic;
private static final String GROUPID = "groupA";
private static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
private static final String DB_URL = "jdbc:mysql://localhost:3306/hse_db_test?useUnicode=true&characterEncoding=utf8&serverTimezone=GMT%2B8&useSSL=false";
// 数据库的用户名与密码，需要根据自己的设置
static final String USER = "root";
static final String PASS = "mysql@1234";
public KafkaConsumerTest(String topicName) {
Properties props = new Properties();
props.put("bootstrap.servers", "172.20.10.3:9092");
props.put("group.id", GROUPID);
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
props.put("session.timeout.ms", "30000");
props.put("auto.offset.reset", "earliest");
props.put("key.deserializer", StringDeserializer.class.getName());
props.put("value.deserializer", StringDeserializer.class.getName());
this.consumer = new KafkaConsumer<String, String>(props);
this.topic = topicName;
this.consumer.subscribe(Arrays.asList(topic));
}
@Override
public void run() {
int messageNo = 1;
System.out.println("---------开始消费---------");
try {
for (;;) {
msgList = consumer.poll(1000);
System.out.println("数据条数："+msgList.count());
if(null!=msgList&&msgList.count()>0){
for (ConsumerRecord<String, String> record : msgList) {
System.out.println(record.value());
//消费100条就打印 ,但打印的数据不一定是这个规律的
if(messageNo%100==0){
System.out.println(messageNo+"=======receive: key = " + record.key() + ", value = " + record.value()+" offset==="+record.offset());
}
//当消费了1000条就退出
if(messageNo%1000==0){
break;
}
messageNo++;
}
}else{
Thread.sleep(3000);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
consumer.close();
}
}
//连接mysql
public static Connection getConnection(){
Connection conn = null;
try {
Class.forName(JDBC_DRIVER);
// 打开链接
System.out.println("连接数据库...");
conn = DriverManager.getConnection(DB_URL,USER,PASS);
return conn;
}catch (SQLException se){
// 处理 JDBC 错误
se.printStackTrace();
}catch(Exception e){
// 处理 Class.forName 错误
e.printStackTrace();
}
return null;
}
public static void main(String args[]) {
KafkaConsumerTest test1 = new KafkaConsumerTest("example");
Thread thread1 = new Thread(test1);
thread1.start();
}
}
在数据库分别对某张表执行curd操作，控制台输出如下:
下载地址：
https://download.csdn.net/download/weixin_44466075/15901156
欢迎留言讨论
439
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
