同步资源:https://github.com/wzq1990413/canal/releases/tag/adapter-clickhouse
参考配置:https://blog.csdn.net/weixin_43975771/article/details/117231605
canal数据同步
git地址:https://github.com/alibaba/canal
1.mysql log_bin检查
SHOW VARIABLES LIKE '%log_bin%'
2.创建canal用户
create user canal@'%'IDENTIFIED WITH mysql_native_password BY 'canal';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT,SUPER ON *.* TO 'canal'@'%';
FLUSH PRIVILEGES;
3.mysql记录文件名与位置
show master status;
4.进入canal目录,修改配置文件
## mysql serverId , v1.0.26+ will autoGen
# canal.instance.mysql.slaveId=0
# enable gtid use true/false
canal.instance.gtidon=false
# position info
canal.instance.master.address=rm-sdfsfsfsf.mysql.rds.sfsdfs.com:3306 #mysql地址
canal.instance.master.journal.name=mysql-bin.008426 #binlog文件
canal.instance.master.position=273824 #binlog位置
canal.instance.master.timestamp= #时间可以不写
canal.instance.master.gtid=
# rds oss binlog
canal.instance.rds.accesskey=
canal.instance.rds.secretkey=
canal.instance.rds.instanceId=
# table meta tsdb info
canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal
# username/password
canal.instance.dbUsername=abc #mysql 用户名
canal.instance.dbPassword=adf!canal #密码
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
MySQL通过canal同步数据到elasticsearch
参考:
https://www.cnblogs.com/sddll/p/16179532.html
https://blog.csdn.net/Le_OOP/article/details/124724362
https://blog.csdn.net/liu1430016196/article/details/126408784 通过Canal将MySQL数据同步到Elasticsearch
https://blog.csdn.net/a1091758887/article/details/126041481 canal1.16同步mysql数据到elasticsearch
https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.4.1-linux-x86_64.tar.gz
可用包:x.x.x.50 服务器 /data/software/canal/canal1.5/canal-es7-main
对应GIT:https://github.com/baishui1818/canal-es7
配置环境变量
[root@test opt]# vim /etc/security/limits.conf
# 配置进程 和线程数
* soft nproc 131072
* hard nproc 131072
#配置 文件句柄数
* soft nofile 65536
* hard nofile 131072
#配置 内存锁定交换
* soft memlock unlimited
* hard memlock unlimited
[root@test opt]# vi /etc/sysctl.conf
#禁用内存与磁盘交换
vm.swappiness=1
#设置虚拟机内存大小
vm.max_map_count=262144
执行 sysctl -p 使用配置生效
[root@test opt]# sysctl -p
安装elasticsearch
创建elasticsearch用户(es不允许 root用户启动 )
[root@test opt]# useradd elasticsearch
解压软件 elasticsearch-7.17.2-linux-x86_64.tar.gz
[root@test opt]# tar -xf elasticsearch-7.17.2-linux-x86_64.tar.gz -C /home/elasticsearch
[root@test opt]# chown -R elasticsearch: /home/elasticsearch/elasticsearch-7.17.2
配置
[root@test opt]# su - elasticsearch
[elasticsearch@test ~]# cd elasticsearch-7.17.2
[elasticsearch@cdh-01 elasticsearch-7.17.2]$ vim config/elasticsearch.yml
cluster.name: my-application
node.name: node-1
path.data: /home/elasticsearch/elasticsearch-7.17.2/data
path.logs: /home/elasticsearch/elasticsearch-7.17.2/logs
bootstrap.memory_lock: false
network.host: 192.168.1.11
cluster.initial_master_nodes: ["node-1"]
http.cors.enabled: true
http.cors.allow-origin: "*"
启动es
[elasticsearch@cdh-01 elasticsearch-7.17.2]$ ./bin/elasticsearch #不报错就运行成功
##[elasticsearch@cdh-01 elasticsearch-7.17.2]$ ./bin/elasticsearch -d ##后台运行
浏览器输入192.168.1.11:9200,验证es
#############################
docker run --name canal-server -p 5602:5602 -d canal/canal-server:v1.1.5 -d