官方资源
https://github.com/alibaba/canal/wiki/QuickStart
https://github.com/alibaba/canal/wiki/Sync-ES
准备
ElasticSearch6.8.x,Mysql5.7
在Mysql中新建库zaowu,再建表
CREATE TABLE `shop` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) DEFAULT NULL,
`tags` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
Mysql中新建用户,用于canal复制数据
创建用户和密码(%外网访问权限,localhost本地访问权限)
create user 'canal'@'%' IDENTIFIED by 'Canal@2020';
grant select,replication slave,replication client on *.* to 'canal'@'%' identified by 'Canal@2020';
grant select,replication slave,replication client on *.* to 'canal'@'localhost' identified by 'Canal@2020';
刷新权限 flush privileges;
新建ES Index、type
PUT http://192.168.200.130:9200/shop-index
{
"mappings": {
"shop": {
"properties": {
"id": {
"type": "integer"
},
"name": {
"type": "text"
},
"tags": {
"type": "text"
}
}
}
}
}
调整mysql配置参考官网
canal.deployer配置如下
vi canal/conf/example/instance.properties
# 与mysql配置中的serverId不同即可
canal.instance.mysql.slaveId=2
# enable gtid use true/false
canal.instance.gtidon=false
canal.instance.master.address=127.0.0.1:3306
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=Canal@2020
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
# table regex
canal.instance.filter.regex=.*\\..*
# table black regex
canal.instance.filter.black.regex=
canal-adapter配置如下
vi canal-adapter/conf/application.yml
server:
port: 8081
spring:
jackson:
date-format: yyyy-MM-dd HH:mm:ss
time-zone: GMT+8
default-property-inclusion: non_null
canal.conf:
mode: tcp
canalServerHost: 127.0.0.1:11111
batchSize: 500
syncBatchSize: 1000
retries: 0
timeout:
accessKey:
secretKey:
srcDataSources:
defaultDS:
url: jdbc:mysql://127.0.0.1:3306/zaowu?useUnicode=true
username: canal
password: Canal@2020
canalAdapters:
- instance: example
groups:
- groupId: g1
outerAdapters:
- name: es
hosts: 192.168.200.130:9300
properties:
cluster.name: elasticsearch
在canal-adapter/conf/es下新建shop.yml
dataSourceKey: defaultDS
destination: example
groupId: g1
esMapping:
_index: shop-index
_type: shop
_id: _id
upsert: true
# pk: id
sql: "select a.id as _id, a.name as name, a.tags as tags from shop a"
# objFields:
# _labels: array:;
# etlCondition: "where a.c_time>={}"
commitBatch: 3000
#启动canal和canal-adapter
sh canal/bin/startup.sh
sh canal-adapter/bin/startup.sh
shop表添加修改数据查看ES中是否同步
tail -f canal-adapter/logs/adapter.log 监视数据是否到canal
http://192.168.200.130:9200/shop-index/_search 查看数据是否到ES