部署Canal同步数据到Elasticsearch

参考: https://juejin.cn/post/6891435372824395784
canal：模式模拟mysql主从复制
主要模块
canal-server(canal_deployer) ：把自己伪装成从库，接收Mysql的同步不数据
canal-adapter：相当于canal的客户端，会从canal-server中获取数据，然后对数据进行同步，可以同步到MySQL、Elasticsearch和HBase等存储中去。
canal-admin： 为canal提供整体配置管理、节点运维等面向运维的功能

1. 数据库准备
mysql 开启binlog,修改格式为ROW
​ 参考: https://www.jianshu.com/p/8e7e288c41b1
​ 登录mysql服务器
​ 使用 mysql --help | grep 'Default options' -A 1 可以查看文件位置
在 /etc/my.cnf中添加 要添加在 [mysqld] 下

log-bin = /var/lib/mysql-files/bin-log
server-id = 1
transaction_isolation = repeatable-read
binlog_format=ROW
#设置binlog清理时间
expire_logs_days = 7
#binlog每个日志文件大小
max_binlog_size = 100m
#binlog缓存大小
binlog_cache_size = 4m
#最大binlog缓存大小
max_binlog_cache_size = 512m

查看权限 ls -ld /var/lib/mysql-files (没有文件夹要创建文件夹)
如果只有root 没有mysql 修改权限 chown mysql.mysql /proc/meminfo
报错查看日志 查看log tail -n 200 /var/log/mysqld.log

重启数据库:
sudo systemctl stop mysqld
sudo systemctl start mysqld

查看数据库是否开启binlog,查看模式

SHOW VARIABLES LIKE 'log_bin'
show global variables like "binlog%";

创建用户
查看用户 select host, user, authentication_string from mysql.user
创建用户: create user “canal”@"%" identified by “canal的密码”
查看用户权限: show grants for “canal”@"%"
给用户赋权: grant SELECT, REPLICATION SLAVE, REPLICATION CLIENT on . to ‘canal’@’%’ identified by “canal的密码”

2.安装canal

1. 下载canal
   canal github地址: https://github.com/alibaba/canal/releases
   直接下载比较慢 可以使用下载加速 下载加速使用 https://d.serctl.com/
   然后使用wget 类似 wget https://dl2.serctl.com/downloads7/2021-03-31-13-31-57-download-canal.admin-1.1.5-SNAPSHOT.tar.gz
   要下载 canal_adapter,canal_deployer (canal_server) (canal-amin 我没用到)
   然后放到各自的文件夹下解压 tar -zxvf canal.admin-1.1.5.tar.gz
   启动在各自bin目录下

3 配置文件
可以参考example中的配置, 我这里有一份配置,可以换成自己的地址即可,
先启动deployer 后启动 adapter 日志在各自的logs 目录下

canal_deployer的conf目录下创建 resourcesearch目录
放入此份文件
instance.properties

#################################################
## mysql serverId , v1.0.26+ will autoGen
# canal.instance.mysql.slaveId=0

# enable gtid use true/false
canal.instance.gtidon=false

# position info
canal.instance.master.address=XXXXXXX #127.0.0.1:3306
canal.instance.master.journal.name=
#mall-mysql-bin.000001
canal.instance.master.position=4
canal.instance.master.timestamp=
canal.instance.master.gtid=

# rds oss binlog
canal.instance.rds.accesskey=
canal.instance.rds.secretkey=
canal.instance.rds.instanceId=

# table meta tsdb info
canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal

#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
#canal.instance.standby.gtid=

# username/password
canal.instance.dbUsername=XXXXXX
canal.instance.dbPassword=XXXXXX
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==

# table regex
canal.instance.filter.regex=XXXXX\\..*,XXXXXXX\\..*  #数据库名称
# table black regex
canal.instance.filter.black.regex=mysql\\.slave_.*
# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch
# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch

# mq config
canal.mq.topic=resourcesearch
# dynamic topic route by schema or table regex
#canal.mq.dynamicTopic=mytest1.user,mytest2\\..*,.*\\..*
canal.mq.partition=0
# hash partition config
#canal.mq.partitionsNum=3
#canal.mq.partitionHash=test.table:id^name,.*\\..*
#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
#################################################

canal_adapter/conf 目录下的application.yml 改为

server:
  port: 8088
spring:
  jackson:
    date-format: yyyy-MM-dd HH:mm:ss
    time-zone: GMT+8
    default-property-inclusion: non_null

canal.conf:
  mode: tcp #tcp kafka rocketMQ rabbitMQ
  flatMessage: true
  zookeeperHosts:
  syncBatchSize: 1000
  retries: 0
  timeout:
  accessKey:
  secretKey:
  consumerProperties:
    # canal tcp consumer
    canal.tcp.server.host: 127.0.0.1:11111
    canal.tcp.zookeeper.hosts:
    canal.tcp.batch.size: 500
    canal.tcp.username:
    canal.tcp.password:
    # kafka consumer
    kafka.bootstrap.servers: 127.0.0.1:9092
    kafka.enable.auto.commit: false
    kafka.auto.commit.interval.ms: 1000
    kafka.auto.offset.reset: latest
    kafka.request.timeout.ms: 40000
    kafka.session.timeout.ms: 30000
    kafka.isolation.level: read_committed
    kafka.max.poll.records: 1000
    # rocketMQ consumer
    rocketmq.namespace:
    rocketmq.namesrv.addr: 127.0.0.1:9876
    rocketmq.batch.size: 1000
    rocketmq.enable.message.trace: false
    rocketmq.customized.trace.topic:
    rocketmq.access.channel:
    rocketmq.subscribe.filter:
    # rabbitMQ consumer
    rabbitmq.host:
    rabbitmq.virtual.host:
    rabbitmq.username:
    rabbitmq.password:
    rabbitmq.resource.ownerId:

  srcDataSources:
    defaultDS:
      url: jdbc:mysql://127.0.0.1:3306/XXXXXXX?useUnicode=true
      username: XXXXX
      password: XXXXXX
  canalAdapters:
  - instance: resourcesearch # canal instance Name or mq topic name
    groups:
    - groupId: g1
      outerAdapters:
      - name: logger
#      - name: rdb
#        key: mysql1
#        properties:
#          jdbc.driverClassName: com.mysql.jdbc.Driver
#          jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true
#          jdbc.username: root
#          jdbc.password: 121212
#      - name: rdb
#        key: oracle1
#        properties:
#          jdbc.driverClassName: oracle.jdbc.OracleDriver
#          jdbc.url: jdbc:oracle:thin:@localhost:49161:XE
#          jdbc.username: mytest
#          jdbc.password: m121212
#      - name: rdb
#        key: postgres1
#        properties:
#          jdbc.driverClassName: org.postgresql.Driver
#          jdbc.url: jdbc:postgresql://localhost:5432/postgres
#          jdbc.username: postgres
#          jdbc.password: 121212
#          threads: 1
#          commitSize: 3000
#      - name: hbase
#        properties:
#          hbase.zookeeper.quorum: 127.0.0.1
#          hbase.zookeeper.property.clientPort: 2181
#          zookeeper.znode.parent: /hbase
      - name: es7
        hosts: XXXXXX:XXXXX # 127.0.0.1:9200 for rest mode es的地址
        properties:
          mode: rest # or rest
          security.auth: xxxxx:xxxxxx #  ES的账号和密码
          #cluster.name: elasticsearch
#        - name: kudu
#          key: kudu
#          properties:
#            kudu.master.address: 127.0.0.1 # ',' split multi address