MySQL 实现将一个库表里面的数据实时更新到另一个库表里面

MySQL 实现将一个库表里面的数据实时更新到另一个库表里面

需求描述:MySQL 里面有很多的数据库,这些数据库里面都有同一种表结构的表 (tb_warn_log),这张表的数据是实时更新的,现在需要将这些表的数据全部实时更新到另一个库的另一张表 (tb_alarm_management) 里面。

数据库结构如下:

tb_warn_log 表结构如下:

tb_alarm_management 表结构如下:

MySQL 存储过程定时任务:

-- 存储过程创建
use mdm_common_db;
delimiter //
create procedure timing_import_proce()
begin
    DECLARE result_code INTEGER DEFAULT 0; -- 定义返回结果并赋初值 0
    DECLARE CONTINUE HANDLER FOR SQLEXCEPTION SET result_code=1; -- 在执行过程中出任何异常设置 result_code 为 1
    DECLARE CONTINUE HANDLER FOR NOT FOUND SET result_code = 2; -- 如果表中没有下一条数据则置为 2
    
        START TRANSACTION; -- 开始事务
        
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,10,' 上海 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 上海铁路局 ',' 杭州工务段 ',' 杭州钱塘江大桥 ','mdm_1_db' from mdm_1_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_1_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_1_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,6,' 郑州 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 郑州铁路局 ',' 洛阳工务段 ',' 洛阳邙山隧道 ','mdm_2_db' from mdm_2_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_2_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_2_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,21,' 麻城 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 武汉铁路局 ',' 麻城工务段 ',' 麻武线 K71'  ,'mdm_3_db' from mdm_3_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_3_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_3_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,11,' 武汉 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 武汉铁路局 ',' 武汉桥工段 ',' 孝感大桥 京广线上行 1136+880','mdm_4_db' from mdm_4_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_4_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_4_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,14,' 广州 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 广州铁路局 ',' 怀化工务段 ',' 沪昆线 1592+130-160 上行 ','mdm_14_db' from mdm_14_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_14_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_14_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,11,' 武汉 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 武汉铁路局 ',' 荆门桥工段 ',' 宜万线 1296+120-300','mdm_15_db' from mdm_15_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_15_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_15_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,4,' 北京 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 北京铁路局 ',' 北京工务段 ',' 北京西站监测站 ','mdm_17_db' from mdm_17_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_17_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_17_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,51,' 青藏 (西宁)',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 青藏铁路局 ',' 西宁工务段 ',' 米拉湾隧道 ','mdm_18_db' from mdm_18_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_18_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_18_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,51,' 青藏 (西宁)',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 青藏铁路局 ',' 西宁工务段 ',' 付家寨三号隧道 ','mdm_19_db' from mdm_19_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_19_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_19_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,12,' 长沙 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 广州铁路局 ',' 长沙高铁工务段 ',' 艾家冲隧道 ','mdm_21_db' from mdm_21_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_21_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_21_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,10,' 上海 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 上海铁路局 ',' 杭州工务段 ',' 杭州铁路宏远 ', 'mdm_55_db' from mdm_55_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_55_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_55_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,21,' 麻城 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 武汉铁路局 ',' 麻城工务段 ',' 武汉局麻城工务段试点项目 ','mdm_64_db' from mdm_64_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_64_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_64_db'); 
        INSERT INTO mdm_common_db.tb_alarm_management(key_name,warn_type,warn_level,alarm_time,alarm_cityId,alarm_cityName,alarm_user,state,ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,TLJ,GWD,JCZ,GWDDatabase) SELECT SensorType,WarnMainType,WarnLevel,Time,11,' 武汉 ',DealName,' 未处理 ',ID,SensorID,WarnChildType,SendMessage,SendFlag,Dealed,DealPhone,Note,' 武汉铁路局 ',' 武汉桥工段 ',' 武汉桥工段铁路桥试点项目 ','mdm_65_db' from mdm_65_db.tb_warn_log WHERE not exists (select * from mdm_common_db.tb_alarm_management where mdm_common_db.tb_alarm_management.ID = mdm_65_db.tb_warn_log.ID AND mdm_common_db.tb_alarm_management.GWDDatabase = 'mdm_65_db'); 
   
        IF result_code = 1 THEN -- 可以根据不同的业务逻辑错误返回不同的 result_code,这里只定义了 1 和 0
            ROLLBACK; 
        ELSE 
            COMMIT; 
        END IF;
        select result_code;
end//
delimiter ;


-- 定时任务创建
create event timing_import_event
on schedule every 30 minute     -- 每隔 30 分钟执行一次存储过程
on completion preserve disable
do call timing_import_proce();


-- 定时任务开启
alter event timing_import_event on completion preserve enable; -- 开启定时任务


-- 求和
select 
(select count(*) from mdm_1_db.tb_warn_log)
+(select count(*) from mdm_2_db.tb_warn_log)
+(select count(*) from mdm_3_db.tb_warn_log)
+(select count(*) from mdm_4_db.tb_warn_log)
+(select count(*) from mdm_14_db.tb_warn_log)
+(select count(*) from mdm_15_db.tb_warn_log)
+(select count(*) from mdm_17_db.tb_warn_log)
+(select count(*) from mdm_18_db.tb_warn_log)
+(select count(*) from mdm_19_db.tb_warn_log)
+(select count(*) from mdm_21_db.tb_warn_log)
+(select count(*) from mdm_55_db.tb_warn_log)
+(select count(*) from mdm_64_db.tb_warn_log)
+(select count(*) from mdm_65_db.tb_warn_log) 求和;

select count(*) from mdm_common_db.tb_alarm_management;
  • 0
    点赞
  • 6
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
### 回答1: 首先,您需要配置源数据(即三个MySQL数据)的CDC,以便FlinkCDC可以获取数据的变更信息。然后您可以使用Flink的DataStream API编写Java代码,从源数据读取变更信息,并将其写入目标数据(即一个MySQL数据)。 以下是一个简单的代码示例: ```java // 连接源数据 DataSource<RowData> source = ... // 连接目标数据 JdbcSink sink = ... // 创建数据流 DataStream<RowData> stream = env.addSource(source); // 将数据写入目标数据 stream.addSink(sink); // 启动Flink程序 env.execute("Flink CDC to MySQL"); ``` 请注意,这只是一个简单的代码示例,需要根据您的具体需求进行定制。您可以使用Flink的各种操作,例如map,filter等,对数据进行处理和转换。 ### 回答2: 编写Java代码实现使用Flink CDC将MySQL三个的所有表同步到一个MySQL数据的过程如下: 首先,需要在代码中导入Flink依赖: ```java import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.api.bridge.java.internal.StreamTableEnvironmentImpl; import org.apache.flink.table.descriptors.Schema; import org.apache.flink.table.factories.TableFactory; import org.apache.flink.table.factories.TableFactoryUtil; import org.apache.flink.table.factories.TableSinkFactory; import org.apache.flink.table.factories.TableSourceFactory; import org.apache.flink.table.sources.TableSource; import org.apache.flink.table.sources.TableSourceValidation; import org.apache.flink.types.Row; ``` 然后,可以定义一个方法来创建Flink CDC源和目标的连接: ```java public class FlinkCDCSync { public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); StreamTableEnvironment tableEnv = StreamTableEnvironmentImpl.create(env); String sourceDDL = "CREATE TABLE sourceTable (\n" + " ... // 设置源表结构\n" + ") WITH (\n" + " 'connector' = 'mysql-cdc',\n" + " 'hostname' = 'source_mysql_host',\n" + " 'port' = 'source_mysql_port',\n" + " 'username' = 'source_mysql_username',\n" + " 'password' = 'source_mysql_password',\n" + " 'database-name' = 'source_database_name',\n" + " 'table-name' = 'source_table_name'\n" + ")"; String sinkDDL = "CREATE TABLE sinkTable (\n" + " ... // 设置目标表结构\n" + ") WITH (\n" + " 'connector' = 'jdbc',\n" + " 'url' = 'jdbc:mysql://sink_mysql_host:sink_mysql_port/sink_database_name',\n" + " 'username' = 'sink_mysql_username',\n" + " 'password' = 'sink_mysql_password',\n" + " 'table-name' = 'sink_table_name'\n" + ")"; tableEnv.executeSql(sourceDDL); tableEnv.executeSql(sinkDDL); String syncSQL = "INSERT INTO sinkTable SELECT * FROM sourceTable"; tableEnv.executeSql(syncSQL); env.execute("Flink CDC Sync"); } } ``` 最后,根据你的需求,创建多个`sourceDDL`和一个`sinkDDL`,分别为每个中的每个表和目标表定义相应的DDL,然后根据需要执行相应的同步操作。 这样,使用Flink CDC就可以将MySQL三个的所有表同步到一个MySQL数据中。注意要根据实际情况替换连接信息和表结构。 ### 回答3: 编写Java代码使用Flink CDC将MySQL三个的所有表同步到一个MySQL数据里面的步骤如下: 1. 导入所需的依赖 ``` <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-jdbc_2.11</artifactId> <version>1.13.0</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-jdbc</artifactId> <version>1.13.0</version> </dependency> ``` 2. 创建Flink CDC的源,连接到MySQL数据中的三个 ```java JdbcCDCSource<RowData> source = JdbcCDCSource.<RowData>builder() .hostname("localhost") .port(3306) .databaseList("db1, db2, db3") .tableList("*") .username("username") .password("password") .deserializer(new RowDataDebeziumDeserializeSchema()) .build(); ``` 3. 创建Flink的执行环境 ```java StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); // 设置并行度 env.addSource(source) .addSink(createJdbcSink()); // 创建MySQL的Sink,将数据写入到目标MySQL数据 env.execute("Flink CDC MySQL Sync"); ``` 4. 创建MySQL的Sink,用于将数据写入到目标MySQL数据 ```java private static JdbcSink<RowData> createJdbcSink() { String insertQuery = "INSERT INTO destination_table (id, name) VALUES (?, ?)"; JdbcStatementBuilder<RowData> statementBuilder = (ps, rowData) -> { ps.setInt(1, rowData.getInt(0)); // 设置需要插入的字段索引和值 ps.setString(2, rowData.getString(1)); }; JdbcExecutionOptions executionOptions = JdbcExecutionOptions.builder() .withBatchSize(1000) .withBatchIntervalMs(100) .build(); JdbcSink<RowData> jdbcSink = JdbcSink.sink( insertQuery, statementBuilder, executionOptions, new JdbcConnectionOptions.JdbcConnectionOptionsBuilder() .withUrl("jdbc:mysql://localhost:3306/destination_db") .withDriverName("com.mysql.jdbc.Driver") .withUsername("username") .withPassword("password") .build() ); return jdbcSink; } ``` 以上是使用Java代码编写的一个简单示例,将MySQL三个的所有表同步到一个MySQL数据中。你可以根据实际需要进行调整和优化。注意将代码中的"hostname"、"port"、"username"、"password"、"db1"、"db2"、"db3"、"destination_table"、"destination_db"等参数替换为实际的数据连接信息和表名称。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值