flink读取并实时监控mysql的变化到kafka里

运行条件

1. mysql的binlog要开启(8.0默认是启动的，其它版本的百度百度吧)
2. zookeeper和Kafka要开启

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.jianxin</groupId>
    <artifactId>work_requirement</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <java.version>1.8</java.version>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <spring-boot.version>2.4.1</spring-boot.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
        <flink.version>1.12.0</flink.version>
        <scala.version>2.12</scala.version>
        <hadoop.version>3.1.3</hadoop.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-cep_${scala.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.68</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba.ververica</groupId>
            <artifactId>flink-connector-mysql-cdc</artifactId>
            <version>1.2.0</version>
        </dependency>

        <!--如果保存检查点到hdfs上，需要引入此依赖-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

        <!--Flink默认使用的是slf4j记录日志，相当于一个日志的接口,我们这里使用log4j作为具体的日志实现-->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>1.7.25</version>
        </dependency>

        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.25</version>
        </dependency>

        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-to-slf4j</artifactId>
            <version>2.14.0</version>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
    </dependencies>

    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-dependencies</artifactId>
                <version>${spring-boot.version}</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>

        </plugins>
    </build>

</project>

FlinkCDC.java

import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import com.alibaba.ververica.cdc.connectors.mysql.MySQLSource;
import com.alibaba.ververica.cdc.connectors.mysql.table.StartupOptions;
import com.alibaba.ververica.cdc.debezium.DebeziumSourceFunction;


public class FlinkCDC {

        public static void main(String[] args) throws Exception {

            //1.创建执行环境
            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
            env.setParallelism(1);

            //2.创建Flink-MySQL-CDC的Source
            DebeziumSourceFunction<String> mysqlSource = MySQLSource.<String>builder()
                    .hostname("localhost")
                    .port(3307)
                    .username("root")
                    .password("chickenkang.")
                    .databaseList("practice")
                    .tableList("practice.user")
                    .serverTimeZone("Asia/Shanghai")
//                    .startupOptions(StartupOptions.latest())
                    .startupOptions(StartupOptions.earliest())  //earlist是从binlog第一行开始  但是earlist有个限制就是你必须在建库之前就开启binlog  如果你是中途开启binlog earlist会有问题
//                    .startupOptions(StartupOptions.initial())
//                .startupOptions(KafkaOptions.StartupOptions.class)
                    .deserializer(new StringDebeziumDeserializationSchema())//官方告诉需要反序列化
                    .deserializer(new CustomerDeserialization())
                    .build();

            //3.使用CDC Source从MySQL读取数据
            DataStreamSource<String> mysqlDS = env.addSource(mysqlSource);
//            DataStreamSource<String> mysqlDS = env.addSource(mysqlSource);
            mysqlDS.print();
            //4.打印数据

            mysqlDS.addSink(MyKafkaUtil.getKafkaSink("ods_base_db"));

            //5.执行任务
            env.execute("flinkcdc");
        }
}

MyKafkaUtil.java

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import org.apache.kafka.clients.consumer.ConsumerConfig;

import java.util.Properties;

public class MyKafkaUtil {
    private static String KAFKA_SERVER = "hadoop01:9092,hadoop02:9092,hadoop03:9092";//Kafka的集群和端口
    private static Properties properties =  new Properties();
    static {
        properties.setProperty("bootstrap.servers",KAFKA_SERVER);
    }
    //返回一个kafka生产者
    public static FlinkKafkaProducer<String> getKafkaSink(String topic){
        return new FlinkKafkaProducer<String>(topic,new SimpleStringSchema(),properties);
    }
    // //返回一个kafka消费者
    public static FlinkKafkaConsumer<String> getKafkaConsumer(String topic,String groupId){
        Properties properties = new Properties();
        properties.put(ConsumerConfig.GROUP_ID_CONFIG,groupId);
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,KAFKA_SERVER);
        return new FlinkKafkaConsumer<String>(topic,new SimpleStringSchema(),properties);

    }
}

CustomerDeserialization.java

import com.alibaba.fastjson.JSONObject;
import com.alibaba.ververica.cdc.debezium.DebeziumDeserializationSchema;
import io.debezium.data.Envelope;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.util.Collector;
import org.apache.kafka.connect.data.Field;
import org.apache.kafka.connect.data.Schema;
import org.apache.kafka.connect.data.Struct;
import org.apache.kafka.connect.source.SourceRecord;

import java.util.List;

public class CustomerDeserialization implements DebeziumDeserializationSchema<String> {
    //自定义数据解析器
    @Override
    public void deserialize(SourceRecord sourceRecord, Collector<String> collector) throws Exception {
        //获取主题信息,包含着数据库和表名  mysql_binlog_source.gmall-flink.z_user_info
        String topic = sourceRecord.topic();
        String[] arr = topic.split("\\.");
        String db = arr[1];
        String tableName = arr[2];
        //获取操作类型 READ DELETE UPDATE CREATE
        Envelope.Operation operation = Envelope.operationFor(sourceRecord);//before空的是因为插入 after是空的是因为删除 修改才是都不为空
        String type = operation.toString().toLowerCase();
        if("create".equals(type)){
            type = "insert";
        }
        //获取值信息并转换为Struct类型
        Struct value = (Struct) sourceRecord.value();
        //3.获取“befor”数据
        Struct before = value.getStruct("before");//
        JSONObject beforeJson = new JSONObject();
        if(before !=null) {
            Schema beforeSchema = before.schema();//字段
            List<Field> beforeFields = beforeSchema.fields();//存储在列表
            for (Field field : beforeFields) {  //field 就是 id  name    beforeValue就是id的值，name的值
                Object beforeValue = before.get(field);
                beforeJson.put(field.name(), beforeValue);
            }
        }
        //4.获取“after”数据
        Struct after = value.getStruct("after");//
        JSONObject afterJson = new JSONObject();
        if(after !=null) {
            Schema afterSchema = after.schema();//字段
            List<Field> afterFields = afterSchema.fields();//存储在列表
            for (Field field : afterFields) {  //field 就是 id  name    afterValue就是id的值，name的值
                Object afterValue = after.get(field);
                afterJson.put(field.name(), afterValue);
            }
        }
        //创建JSON对象用于封装最终返回值数据信息
        JSONObject result = new JSONObject();
        result.put("operation", operation.toString().toLowerCase());
        result.put("before", beforeJson);
        result.put("after", afterJson);
        result.put("database", db);
        result.put("table", tableName);
        //发送数据至下游
        collector.collect(result.toJSONString());
    }
    @Override
    public TypeInformation<String> getProducedType() { return TypeInformation.of(String.class); }
}