第三章 FlinkCDC专题之Mysql-CDC-source基础练习（stream形式）

1、导入依赖

<dependencies>
 <dependency>
 <groupId>org.apache.flink</groupId>
 <artifactId>flink-java</artifactId>
 <version>1.12.0</version>
 </dependency>
 <dependency>
 <groupId>org.apache.flink</groupId>
 <artifactId>flink-streaming-java_2.12</artifactId>
 <version>1.12.0</version>
 </dependency>
 <dependency>
 <groupId>org.apache.flink</groupId>
 <artifactId>flink-clients_2.12</artifactId>
 <version>1.12.0</version>
 </dependency>
 <dependency>
 <groupId>org.apache.hadoop</groupId>
 <artifactId>hadoop-client</artifactId>
 <version>3.1.3</version>
 </dependency>
 <dependency>
 <groupId>mysql</groupId>
 <artifactId>mysql-connector-java</artifactId>
 <version>5.1.49</version>
</dependency>
<dependency>
 <groupId>org.apache.flink</groupId>
 <artifactId>flink-table-planner-blink_2.12</artifactId>
 <version>1.12.0</version>
</dependency>
<dependency>
 <groupId>com.ververica</groupId>
<artifactId>flink-connector-mysql-cdc</artifactId>
 <version>2.0.2</version>
</dependency>
 <dependency>
 <groupId>com.alibaba</groupId>
 <artifactId>fastjson</artifactId>
 <version>1.2.75</version>
 </dependency>
</dependencies>
<build>
 <plugins>
 <plugin>
 <groupId>org.apache.maven.plugins</groupId>
 <artifactId>maven-assembly-plugin</artifactId>
 <version>3.0.0</version>
 <configuration>
 <descriptorRefs>
 <descriptorRef>jar-with-dependencies</descriptorRef>
 </descriptorRefs>
 </configuration>
 <executions>
 <execution>
 <id>make-assembly</id>
 <phase>package</phase>
 <goals>
 <goal>single</goal>
 </goals>
 </execution>
 </executions>
 </plugin>
 </plugins>
</build>

2、编写脚本

package com.hxjy;

import com.ververica.cdc.connectors.mysql.MySqlSource;
import com.ververica.cdc.connectors.mysql.table.StartupOptions;
import com.ververica.cdc.debezium.DebeziumSourceFunction;
import com.ververica.cdc.debezium.StringDebeziumDeserializationSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class mysqldemo {
    public static void main(String[] args) throws  Exception{

        // 1、构建环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        // 2、从创建MysqlCDC的Source
        DebeziumSourceFunction<String> mySqlSource = MySqlSource.<String>builder()
                .hostname("hadoop102")
                .port(3306)
                .databaseList("flink") // set captured database
                .tableList("flink.stu") // set captured table
                .username("root")
                .password("123456")
                .startupOptions(StartupOptions.initial())
                .deserializer(new StringDebeziumDeserializationSchema())
                .build();

        // 3、使用CDC Source方式从mysql中读取数据
        DataStreamSource<String> mysqlDS = env.addSource(mySqlSource);

        // 4、打印数据
        mysqlDS.print();

        // 5、执行任务
        env.execute();
    }
}

3、测试环境准备

• 在mysql中对数据监测的库开启binlog

sudo vim /etc/my.cnf

#添加数据库的binlog
binlog-do-db=flink

#重启MySQL服务
sudo systemctl restart mysqld

• 查询生成日志

cd /var/lib/mysql

• 创建库表并插入数据

create database flink;

use flink;

CREATE TABLE `stu` (
  `id` int(5),
  `name` varchar(20) ,
  `grade` int(5),
  `score` int(5),
    primary key (id)
) 


insert into stu value(1,"zhangsan",9,99);

.4、本地连接测试

（1）运行程序查看日志

 SourceRecord{sourcePartition={server=mysql_binlog_source}, sourceOffset={ts_sec=1640779871, file=mysql-bin.000001, pos=881}} ConnectRecord{topic='mysql_binlog_source.flink.stu', kafkaPartition=null, key=null, keySchema=null, value=Struct{after=Struct{id=1,name=zhangsan,grade=9,score=99},source=Struct{version=1.5.2.Final,connector=mysql,name=mysql_binlog_source,ts_ms=1640779871581,snapshot=last,db=flink,table=stu,server_id=0,file=mysql-bin.000001,pos=881,row=0},op=r,ts_ms=1640779871585}, valueSchema=Schema{mysql_binlog_source.flink.stu.Envelope:STRUCT}, timestamp=null, headers=ConnectHeaders(headers=)}

（2）在mysql中对数据进行增删改查

#插入数据
insert into stu value(2,"wangwu",8,88);
#更新数据
update stu set grade = 10,score=100 where id = 2;
#删除数据
delete from stu where id = 2;

（3）运行程序，查看日志 - - curd