Flink cdc 实现源表sqlserver到目标表sqlserver或者mysql

说明，本文章只提供借鉴，博主也是初次接触flink，请见谅

flink版本1.17.0，flink cdc版本2.3.0，百分百可以实时同步，亲测可用

首先看下我的程序结构吧

准备

1、依赖

    <properties>
        <flink.version>1.17.0</flink.version>
        <flink.cdc.version>2.3.0</flink.cdc.version>
        <logback-classic.version>1.1.11</logback-classic.version>
        <slf4j-api.version>1.7.25</slf4j-api.version>
    </properties>

    <!-- 依赖器 -->
    <dependencyManagement>
        <dependencies>

            <!-- SpringCloud 微服务 -->
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-dependencies</artifactId>
                <version>2021.0.4</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>

            <!-- SpringCloud Alibaba 微服务 -->
            <dependency>
                <groupId>com.alibaba.cloud</groupId>
                <artifactId>spring-cloud-alibaba-dependencies</artifactId>
                <version>2021.0.4.0</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>

            <!-- SpringBoot 依赖配置 -->
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-dependencies</artifactId>
                <version>2.7.3</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>

    <dependencies>
        <!-- bootstrap 启动器 -->
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-bootstrap</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
            <version>2.7.3</version>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-freemarker</artifactId>
            <version>2.4.4</version>
        </dependency>

        <dependency>
            <groupId>org.mybatis</groupId>
            <artifactId>mybatis</artifactId>
            <version>3.5.9</version>
        </dependency>

        <dependency>
            <groupId>com.microsoft.sqlserver</groupId>
            <artifactId>mssql-jdbc</artifactId>
            <version>9.4.1.jre8</version>
        </dependency>

        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.8.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        
        <dependency>
            <groupId>com.ververica</groupId>
            <artifactId>flink-connector-sqlserver-cdc</artifactId>
            <version>${flink.cdc.version}</version>
        </dependency>

        <dependency>
            <groupId>com.ververica</groupId>
            <artifactId>flink-connector-oracle-cdc</artifactId>
            <version>${flink.cdc.version}</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.33</version>
        </dependency>

        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-context-support</artifactId>
            <version>5.3.22</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-runtime-web</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>com.oracle.database.jdbc</groupId>
            <artifactId>ojdbc8</artifactId>
            <version>12.2.0.1</version>
        </dependency>

        <dependency>
            <groupId>com.ibm.db2</groupId>
            <artifactId>jcc</artifactId>
            <version>11.5.0.0</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.26</version>
            <scope>compile</scope>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-configuration-processor</artifactId>
            <optional>true</optional>
        </dependency>

    </dependencies>

2、打包插件

    <build>
            <plugins>
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-compiler-plugin</artifactId>
                    <version>3.1</version>
                    <configuration>
                        <source>1.8</source>
                        <target>1.8</target>
                    </configuration>
                </plugin>
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-shade-plugin</artifactId>
                    <version>3.1.1</version>
                    <executions>
                        <!-- Run shade goal on package phase -->
                        <execution>
                            <phase>package</phase>
                            <goals>
                                <goal>shade</goal>
                            </goals>
                            <configuration>
                                <artifactSet>
                                    <excludes>
                                        <exclude>org.apache.flink:force-shading</exclude>
                                        <exclude>com.google.code.findbugs:jsr305</exclude>
<!--                                        <exclude>org.slf4j:*</exclude>-->
<!--                                        <exclude>org.apache.logging.log4j:*</exclude>-->
                                    </excludes>
                                </artifactSet>
                                <filters>
                                    <filter>
                                        <artifact>*:*</artifact>
                                        <excludes>
                                            <exclude>META-INF/*.SF</exclude>
                                            <exclude>META-INF/*.DSA</exclude>
                                            <exclude>META-INF/*.RSA</exclude>
                                        </excludes>
                                    </filter>
                                </filters>
                                <transformers combine.children="append">
                                    <!-- TODO：这个防止多个connector的相同类名覆盖-->
                                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                                    <!--指定 主类-->
                                    <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                        <mainClass>com.hbos.FlinkCDCApplication</mainClass>
                                    </transformer>
                                </transformers>
                            </configuration>
                        </execution>
                    </executions>
                </plugin>
            </plugins>
            <resources>
                <resource>
                    <directory>src/main/resources</directory>
                    <includes>
                        <include>application.yml</include>
                        <include>**/*.ftl</include>
                        <include>**/*.xml</include>
                        <include>**/*.properties</include>
                    </includes>
                    <filtering>true</filtering>
                </resource>
            </resources>
        </build>

编写主类

@SpringBootApplication(excludeName = {"org.springframework.security"})
@Slf4j
public class FlinkCDCApplication {

    public static void main(String[] args){

        SpringApplication.run(FlinkCDCApplication.class,args);
        log.info("CDC服务启动成功");
        System.out.println("=========================="+"\n"+
                "(♥◠‿◠)ﾉﾞCDC服务启动成功"+"\n"
                +"==========================");
    }

}

编写源表SourceConfig 类配置

package com.hbos.cdc.config;

import com.hbos.cdc.entity.SourceDataProperties;
import com.hbos.cdc.listener.DataChangeSink;
import com.ververica.cdc.connectors.sqlserver.SqlServerSource;
import com.ververica.cdc.connectors.sqlserver.table.StartupOptions;
import com.ververica.cdc.debezium.DebeziumSourceFunction;
import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.kafka.connect.json.DecimalFormat;
import org.apache.kafka.connect.json.JsonConverterConfig;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

@Configuration
public class SourceConfig {

    @Bean(name = "sourceInfo")
    @ConfigurationProperties(prefix = "source")
    public SourceDataProperties sourceInfo() {
        return new SourceDataProperties();
    }

    @Bean
    public StreamExecutionEnvironment source(@Qualifier("sourceInfo") SourceDataProperties sourceInfo) throws Exception {


        Map config = new HashMap();
        config.put(JsonConverterConfig.DECIMAL_FORMAT_CONFIG, DecimalFormat.NUMERIC.name());
        JsonDebeziumDeserializationSchema jdd = new JsonDebeziumDeserializationSchema(false, config);

        //通过FlinkCDC构建SourceFunction
        DebeziumSourceFunction<String> sourceFunction  = SqlServerSource.<String>builder()
                .hostname(sourceInfo.getHost())
                .port(sourceInfo.getPort())
                .username(sourceInfo.getUsername())
                .password(sourceInfo.getPassword())
                .database(sourceInfo.getDatabase()) //设置捕获的库名
                .tableList(sourceInfo.getTableList()) //设置捕获的表,也可以去掉表，那就证明监控某库下所有的表
                .deserializer(jdd) //转换成JSON
                /*
                 * initial初始化快照,即全量导入后增量导入(检测更新数据写入)
                 * latest:只进行增量导入(不读取历史变化)
                 */
                .startupOptions(StartupOptions.initial())//启动模式
                .build();

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(5); //全局并行度为5

        env.addSource(sourceFunction).addSink(new DataChangeSink()).setParallelism(2);
        //4、启动任务
        env.execute("JHYGCDC1");
        return env;
    }
}

因为flink的结构和springboot的结构不一样，所以要想把bean交给spring管理那就需要手动去创建

创建bean工具，也就是SpringBeanUtil类

@Component
public class SpringBeanUtil implements ApplicationContextAware {

    private static ApplicationContext applicationContext;

    public void setApplicationContext(ApplicationContext applicationContext)
      throws BeansException {
        SpringBeanUtil.applicationContext = applicationContext;
    }

    public static <T> T getBean(Class<T> clazz) {
        return applicationContext.getBean(clazz);
    }

    public static Object getBean(String name) throws BeansException {

        return applicationContext.getBean(name);
    }
}

创建源表实体和目标表实体

源实体SourceDataProperties类

//@Component
@Data
//@ConfigurationProperties(prefix = "source")
public class SourceDataProperties {

    private String driverClassName;

    private String url;

    private String host;

    private int port;

    private String username;

    private String password;

    private String database;

    private String tableList;

}

目标实体TargetDataProperties 类

@Component
@Data
@ConfigurationProperties(prefix = "target")
public class TargetDataProperties {

    private String driverClassName;

    private String url;

    private String host;

    private String port;

    private String username;

    private String password;

    private String database;

//    @Value("${CDC.DataSource.target.schema}")
//    private String schema;
//
//    @Value("${CDC.DataSource.target.tableList}")
//    private String tableList;

}

sink类

也就是处理业务逻辑的代码，有点多我就不粘贴出来了，你们可以自己处理你们自己的业务逻辑

也可以借鉴我的，地址如下

flink cdc业务逻辑层处理-CSDN博客https://blog.csdn.net/weixin_50448162/article/details/140635326?spm=1001.2014.3001.5502