支持Oracle/MySQL/SQL server/PostgreSQL(同构/异构)数据库同步

本文链接：https://blog.csdn.net/weixin_43965814/article/details/128638382
同构/异构数据库同步/迁移软件--zcbus

zcbus简介

ZCBUS数据服务交换和服务工具作为一体化数据治理和共享平台一个重要组成部分,包含了数据采集服务、数据交换服务、数据加工服务、数据共享服务等模块。数据交换和服务工具采用面向微服务的插座式架构，实现数据交换、数据整合、数据复制、数据传输、数据共享等功能。支持跨组织、跨网段、跨区域、上下级组织之间海量数据交换和服务、适应基于前置机、逻辑隔离的数据交换和服务等多种应用场景，数据总线能够确保数据完整性、安全性、可靠性和传输性能，能够提升数据质量与及应用价值，为数据需求部门提供准确可用的数据。
程序集群架构如下图所示，满足高可用容灾需求。当然基础测试版也可以安装单机模式，安装过程会加在文章后面以供供参考。
zcbus单机组件介绍
1、安装环境要求

内容	要求
操作系统版本	Linux 7 以上
防火墙	关闭
seLinux	关闭
磁盘空间	缓存目录建议同步数据库大小的 70%
2、安装步骤

# 进入到软件解压目录（以实际解压路径为主，此处为演示路径）
cd /home/zcbus/data1/simple_server/
#使用zcmgr.sh安装软件（可使用zcmgr.sh --help查看帮助）
[root@lyzcbus simple_server]# ./zcmgr.sh install

[Step 1]: checking OS version/firewalld/seLinux  and config ...
############################## check host ::: /etc/hosts #############################################
➜ check host ::: 0 ::: WARNING
➜ 
############################## check selinux ::: /etc/selinux/config #################################
➜ check selinux ::: disabled ::: OK
setenforce: SELinux is disabled
############################## check firewalld #######################################################
➜ check firewalld ::: not running ::: OK
############################## check sysctl ::: /etc/sysctl.conf #####################################
➜ kernel.shmall ::: not set (suggest:4294967296) ::: WARNING
➜ fs.aio-max-nr ::: not set (suggest:1048576) ::: WARNING
➜ fs.file-max ::: not set (suggest:6815744) ::: WARNING
➜ kernel.shmmax ::: not set (suggest:2070833152) ::: WARNING
➜ kernel.shmmni ::: not set (suggest:4096) ::: WARNING
➜ kernel.sem ::: not set (suggest:250 32000 100 128) ::: WARNING
➜ net.ipv4.ip_local_port_range ::: not set (suggest:1024 65500) ::: WARNING
➜ net.core.rmem_default ::: not set (suggest:262144) ::: WARNING
➜ net.core.rmem_max ::: not set (suggest:4194304) ::: WARNING
➜ net.core.wmem_default ::: not set (suggest:262144) ::: WARNING
➜ net.core.wmem_max ::: not set (suggest:1048576) ::: WARNING
➜ kernel.threads-max ::: not set (suggest:999999) ::: WARNING
➜ kernel.pid_max ::: not set (suggest:999999) ::: WARNING
➜ vm.max_map_count ::: not set (suggest:1999999) ::: WARNING
➜ net.ipv4.ip_forward ::: not set (suggest:1) ::: WARNING
➜ fs.inotify.max_user_watches ::: not set (suggest:1048576) ::: WARNING
➜ fs.inotify.max_user_instances ::: not set (suggest:1048576) ::: WARNING
############################## check limits ::: /etc/security/limits.conf ############################
➜ soft-nofile ::: not set (suggest:* soft nofile 1048500) ::: WARNING
➜ hard-nofile ::: not set (suggest:* hard nofile 1048500) ::: WARNING
➜ soft-nproc ::: not set (suggest:* soft nproc 65536) ::: WARNING
➜ hard-nproc ::: not set (suggest:* hard nproc 65536) ::: WARNING
############################## CHECK RESULT ##########################################################
➜ OK : 2 ERROR : 1 WARNING : 22
############################## modify command ########################################################
###modify host
echo "127.0.0.1 lyzcbus.test" >> /etc/hosts
###modify selinux
###modify firewalld
###modify sysctl
echo "kernel.shmall = 4294967296" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "fs.file-max = 6815744" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "kernel.shmmax = 2070833152" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "kernel.shmmni = 4096" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "kernel.sem = 250 32000 100 128" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "net.ipv4.ip_local_port_range = 1024 65500" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "net.core.rmem_default = 262144" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "net.core.rmem_max = 4194304" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "net.core.wmem_default = 262144" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "net.core.wmem_max = 1048576" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "kernel.threads-max = 999999" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "kernel.pid_max = 999999" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "vm.max_map_count = 1999999" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "fs.inotify.max_user_watches = 1048576" >> /etc/sysctl.conf;/sbin/sysctl -p
echo "fs.inotify.max_user_instances = 1048576" >> /etc/sysctl.conf;/sbin/sysctl -p
###modify limits
echo "* soft nofile 1048500" >> /etc/security/limits.conf
echo "* hard nofile 1048500" >> /etc/security/limits.conf
echo "* soft nproc 65536" >> /etc/security/limits.conf
echo "* hard nproc 65536" >> /etc/security/limits.conf
kernel.shmall = 4294967296
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
kernel.threads-max = 999999
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
kernel.threads-max = 999999
kernel.pid_max = 999999
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
kernel.threads-max = 999999
kernel.pid_max = 999999
vm.max_map_count = 1999999
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
kernel.threads-max = 999999
kernel.pid_max = 999999
vm.max_map_count = 1999999
net.ipv4.ip_forward = 1
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
kernel.threads-max = 999999
kernel.pid_max = 999999
vm.max_map_count = 1999999
net.ipv4.ip_forward = 1
fs.inotify.max_user_watches = 1048576
kernel.shmall = 4294967296
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 2070833152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 1024 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
kernel.threads-max = 999999
kernel.pid_max = 999999
vm.max_map_count = 1999999
net.ipv4.ip_forward = 1
fs.inotify.max_user_watches = 1048576
fs.inotify.max_user_instances = 1048576
############################## check host ::: /etc/hosts #############################################
➜ check host ::: 1 ::: OK
➜ 127.0.0.1 lyzcbus.test
############################## check selinux ::: /etc/selinux/config #################################
➜ check selinux ::: disabled ::: OK
setenforce: SELinux is disabled
############################## check firewalld #######################################################
➜ check firewalld ::: not running ::: OK
############################## check sysctl ::: /etc/sysctl.conf #####################################
➜ kernel.shmall ::: 4294967296 (>=4294967296) ::: OK
➜ fs.aio-max-nr ::: 1048576 (>=1048576) ::: OK
➜ fs.file-max ::: 6815744 (>=6815744) ::: OK
➜ kernel.shmmax ::: 2070833152 (>=2070833152) ::: OK
➜ kernel.shmmni ::: 4096 (>=4096) ::: OK
➜ kernel.sem ::: 250 (>=250) 32000 (>=32000) 100 (>=100) 128 (>=128) ::: OK
➜ net.ipv4.ip_local_port_range ::: 1024 (>=1024) 65500 (>=65500) ::: OK
➜ net.core.rmem_default ::: 262144 (>=262144) ::: OK
➜ net.core.rmem_max ::: 4194304 (>=4194304) ::: OK
➜ net.core.wmem_default ::: 262144 (>=262144) ::: OK
➜ net.core.wmem_max ::: 1048576 (>=1048576) ::: OK
➜ kernel.threads-max ::: 999999 (>=999999) ::: OK
➜ kernel.pid_max ::: 999999 (>=999999) ::: OK
➜ vm.max_map_count ::: 1999999 (>=1999999) ::: OK
➜ net.ipv4.ip_forward ::: 1 (>=1) ::: OK
➜ fs.inotify.max_user_watches ::: 1048576 (>=1048576) ::: OK
➜ fs.inotify.max_user_instances ::: 1048576 (>=1048576) ::: OK
############################## check limits ::: /etc/security/limits.conf ############################
➜ soft-nofile ::: 1048500 (>=1048500) ::: OK
➜ hard-nofile ::: 1048500 (>=1048500) ::: OK
➜ soft-nproc ::: 65536 (>=65536) ::: OK
➜ hard-nproc ::: 65536 (>=65536) ::: OK
############################## CHECK RESULT ##########################################################
➜ OK : 26 ERROR : 0 WARNING : 0
[INFO] whether to start install zcbus ... Please input 'y/Y' to continue/press Ctrl+C to exit :y
[INFO] whether to install chinese[0]/english[1],defaut is 0:
[INFO] whether to install integrate/external ,defaut is integrate:
####################install module####################
.......................................[ server ]
.......................................[ client ]
.......................................[ server_full ]
.......................................[ client_full ]
.......................................[ cloud_server ]
.......................................[ cloud_client ]
.......................................[ cloud_push ]
Please input Select Mode,default mode is [ server ]:

[Step 2]: ready for data dir path ...
[INFO] Please input docker path /data/docker02:/data
➜ Not Found path /data...
➜ mkdir path for /data finished ...
➜ Find path /data
➜ Ready Path for /data finished ...

[Step 3]: checking if docker is installed ...
➜ unzip soft/docker-20.10.10.tgz ...
docker/
docker/dockerd
docker/docker-init
docker/runc
docker/ctr
docker/containerd-shim-runc-v2
docker/containerd-shim
docker/docker
docker/containerd
docker/docker-proxy
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /etc/systemd/system/docker.service.
Docker version 20.10.10, build b485636
############################## docker version: 20.10.10 ##############################################

[Step 4]: checking docker-compose is installed ...

Install docker-compose ...
docker-compose version 1.25.0-rc4, build 8f3c9c58
############################## docker-compose version: 1.25.0 ########################################

[Step 5]: checking mysql directory ...
############################## Create directory mysql /data/zcbusdata .... ###########################
✔ Mysql data directory  /data/zcbusdata/mysql/data create Successful! 

[Step 6]: checking kafka directory ...
############################## Create directory kafka /data/zcbusdata/kafka .... #####################
✔ Kafka data directory /data/zcbusdata/zcbuskafka/logs create Successful! 

[Step 7]: checking cache directory ...
############################## Create directory cache /data/zcbusdata .... ###########################
✔ Mysql data directory  /data/zcbusdata/cache create Successful! 

[Step 8]: checking zookeeper directory ...
############################## Create directory kafka /data/zcbusdata/kafka .... #####################
✔ Zookeeoer data directory /data/zcbusdata/zcbuszookeeper/data create Successful! 

[Step 9]: loading zcbus images ...
[INFO] Please input if load images y/n:y
docker load -i /soft/simple_server/soft/zcbus.tar.gz
## 挂载镜像过程无需操作，过程已省略
Loaded image: reg.zbomc.com/zcbus_restapi:v2.0.1
➜ Load images to docker's sucessfull...
➜ 
f0bff61e56e1e130e81d598bac564768af80c9b980e5f7f15f5e62a081672bac
➜ network zcbus is create sucessfull...
Check hostname [lyzcbus.test]'s ip is :[lyzcbus.test]
ready basic info ...
Finish init basic ...
➜ prepare server Sucessfull...
➜ 
➜ 

[Step 10]: checking if ports is used ...
➜ Port  3306 is available!!
➜ Port  8890 is available!!

[Step 11]: ready basic soft for container ...
➜ Ready compare soft ...
➜ Read zcbusserver jdk sucessfull....
➜ Read zcbusserver jar sucessfull....
➜ Read zcbusserver bin sucessfull....
➜ Read zcbusserver lib sucessfull....
➜ /data/module/lib to /data/zcbusdata/zcbusserver/ is build sucessfull...
➜ /data/module/bin to /data/zcbusdata/zcbusserver/ is build sucessfull...
➜ /data/module/jdk to /data/zcbusdata/zcbusserver/ is build sucessfull...
➜ /data/module/jar to /data/zcbusdata/zcbusserver/ is build sucessfull...

[Step 12]: starting zcbus ...
Creating zcbusdb        ... done
Creating zcbuszookeeper ... done
Creating zcbusrestapi   ... done
Creating zcbusvue       ... done
Creating zcbuskafka     ... done

====>>>ZCBUS [ Mon Jan 16 14:23:46 CST 2023 ]
     Name                   Command               State                          Ports                       
-------------------------------------------------------------------------------------------------------------
zcbusdb          docker-entrypoint.sh /run.sh     Up      0.0.0.0:3306->3306/tcp,:::3306->3306/tcp, 33060/tcp
zcbuskafka       docker-entrypoint.sh             Up                                                         
zcbusrestapi     sh /run.sh                       Up      7080/tcp                                           
zcbusvue         /docker-entrypoint.sh /bin ...   Up      0.0.0.0:8890->80/tcp,:::8890->80/tcp               
zcbuszookeeper   docker-entrypoint.sh             Up                                                         
➜ 

[Step 12]: ready basic soft for basic zcbus_docker server ...
Check hostname [lyzcbus.test]'s ip is :[lyzcbus.test]

Note: add zcbus_docker service
sed "s#Z_HOME#/soft/simple_server#g;s#Z_IP#lyzcbus.test#g;" /soft/simple_server/common/zcbus_docker.service
? add zcbus_docker service successfully ...
grep db_external_port /soft/simple_server/common/zcbus.yml|awk -F : '{print }' |sed 's/ //g'
➜ Read Master database info to /soft/simple_server/config/zcbus_master.ini
➜ sdbinfo :QFlYT0k6/e0twWGp8aVtWfGB8dn9YdTo@zcbusdb:3306/zcbus...
[LV0] 2023-01-16 14:23:49: connect to mysql zcbus/***@lyzcbus.test:3306 ...
error at file[mysql/mysql.c] line[46], ZCBUS-20000: mysql connect failed: Lost connection to MySQL server at 'reading initial communication packet', system error: 0
error at file[mysql/mysql.c] line[118], ZCBUS-20000: mysql connect failed: Lost connection to MySQL server at 'reading initial communication packet', system error: 0
error at file[db.c] line[426], ZCBUS-20000: mysql connect failed: Lost connection to MySQL server at 'reading initial communication packet', system error: 0
[ERR] connection test failed, ZCBUS-20000: mysql connect failed: Lost connection to MySQL server at 'reading initial communication packet', system error: 0
➜ Test zcbus_docker Connect to zcbus_master.ini failed retry 10 seconds...
Check hostname [lyzcbus.test]'s ip is :[lyzcbus.test]

Note: add zcbus_docker service
grep db_external_port /soft/simple_server/common/zcbus.yml|awk -F : '{print }' |sed 's/ //g'
➜ Read Master database info to /soft/simple_server/config/zcbus_master.ini
➜ sdbinfo :QFlYT0k6/e0twWGp8aVtWfGB8dn9YdTo@zcbusdb:3306/zcbus...
[LV0] 2023-01-16 14:23:59: connect to mysql zcbus/***@lyzcbus.test:3306 ...
[INF] set client character set utf8mb4...
[INF] new client character set: utf8mb4
[INF] MYSQL VERSION: 50732
[INF] MYSQL INFO: 5.7.32-log
SET SESSION sql_mode='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'
[INF] connection test ok.
➜ Test zcbus_docker Connect to zcbus_master.ini Sucessfull...
➜ =========================== ready restart zcbus_docker ====================
zcbus_docker is Stopping ...
zcbus_docker is Finished ...
zcbus_host=lyzcbus.test
zcbus_port=3306
zcbus_user=******
zcbus_password=******
zcbus_db=zcbus

archive log /soft/simple_server/log/log.docker ---> /soft/simple_server/log/archive/log.docker.20230116142359
➜ =========================== ready restart zcbus_docker ====================
zcbus_docker is Stopping ...
kill -9 56053...
zcbus_docker is Finished ...
zcbus_host=lyzcbus.test
zcbus_port=3306
zcbus_user=******
zcbus_password=******
zcbus_db=zcbus

archive log /soft/simple_server/log/log.docker ---> /soft/simple_server/log/archive/log.docker.20230116142400
➜ Manager console Website Address: http://lyzcbus.test:8890
➜ Default login account : admin
➜ Default login password: 123456
➜ 
➜ =======================================
✔ ----Zcbus has been installed and started successfully.----
➜ Ready compare soft ...
➜ /data/module/lib to /data/zcbusdata/zcbus/ is build sucessfull...
➜ /data/module/bin to /data/zcbusdata/zcbus/ is build sucessfull...
➜ /data/module/jdk to /data/zcbusdata/zcbus/ is build sucessfull...
➜ /data/module/jar to /data/zcbusdata/zcbus/ is build sucessfull...
Check hostname [lyzcbus.test]'s ip is :[lyzcbus.test]
Flush data to /compose_location/docker-compose.yml
zcbuszookeeper is up-to-date
zcbusdb is up-to-date
Creating zcbus ... 
zcbusrestapi is up-to-date
zcbusvue is up-to-date
Creating zcbus ... done

====>>>ZCBUS [ Mon Jan 16 14:24:05 CST 2023 ]
     Name                   Command               State                          Ports                       
-------------------------------------------------------------------------------------------------------------
zcbus            /bin/bash -c ${ZCBUS_HOME} ...   Up                                                         
zcbusdb          docker-entrypoint.sh /run.sh     Up      0.0.0.0:3306->3306/tcp,:::3306->3306/tcp, 33060/tcp
zcbuskafka       docker-entrypoint.sh             Up                                                         
zcbusrestapi     sh /run.sh                       Up      7080/tcp                                           
zcbusvue         /docker-entrypoint.sh /bin ...   Up      0.0.0.0:8890->80/tcp,:::8890->80/tcp               
zcbuszookeeper   docker-entrypoint.sh             Up                                                         
root      56078      1  0 14:23 pts/1    00:00:00 /soft/simple_server/bin/zcbus_docker -log_level 2
# 看到如上几个容器信息，软件即为安装成功，访问WEB页面地址为：http://IP:8890