pom文件
<properties>
<flink.version>1.13.6</flink.version>
<scala.binary.version>2.12</scala.binary.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-jdbc_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-json</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<!-- mysql -->
<dependency>
<groupId>com.alibaba.ververica</groupId>
<artifactId>flink-connector-mysql-cdc</artifactId>
<version>1.4.0</version>
</dependency>
<!-- Kafka -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<!-- HBase -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-hbase-2.2_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
<version>2.3.5</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.75</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.20</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<!--如果要打包的话,这里要换成对应的 main class-->
<mainClass>flinksql.flinksql_cdc_kafka</mainClass>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*:*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>6</source>
<target>6</target>
</configuration>
</plugin>
</plugins>
</build>
mysql to kafka
代码里的配置项,例如IP端口需要根据自己的环境修改
package flinksql;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
public class flinksql_cdc_kafka {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
tableEnv.executeSql("create table mysql_cdc(" +
" id INT," +
" username STRING," +
" password STRING," +
" primary key (id) not enforced" +
")WITH(" +
" 'connector' = 'mysql-cdc'," +
" 'hostname' = 'hadoop172'," +
" 'port' = '3306'," +
" 'username' = 'root'," +
" 'password' = '123456'," +
" 'database-name' = 'old_data'," +
" 'table-name' = 'demo'," +
" 'debezium.snapshot.mode' = 'initial'" +
")");
tableEnv.executeSql("CREATE TABLE kafka_mysql_cdc (" +
"id INT," +
"username STRING," +
"password STRING" +
")WITH(" +
" 'connector' = 'kafka'," +
" 'topic' = 'flinkcdc'," +
" 'scan.startup.mode' = 'earliest-offset'," +
" 'properties.bootstrap.servers' = 'hadoop171:9092'," +
" 'properties.group.id' = 'testGroup'," +
" 'format' = 'debezium-json'" +
")");
tableEnv.executeSql("insert into kafka_mysql_cdc select id,username,password from mysql_cdc");
tableEnv.executeSql("select * from mysql_cdc").print();
}
}
mysql to hbase
package flinksql;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
public class flinksql_cdc_hbase {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
tableEnv.executeSql("create table mysql_cdc(" +
" id INT," +
" username STRING," +
" password STRING," +
" primary key (id) not enforced" +
")WITH(" +
" 'connector' = 'mysql-cdc'," +
" 'hostname' = 'hadoop172'," +
" 'port' = '3306'," +
" 'username' = 'root'," +
" 'password' = '123456'," +
" 'database-name' = 'old_data'," +
" 'table-name' = 'demo'," +
" 'debezium.snapshot.mode' = 'initial'" +
")");
tableEnv.executeSql("create table hbase_mysql_cdc(" +
"id INT," +
"info ROW<username STRING,password STRING>," +
"PRIMARY KEY(id) NOT ENFORCED" +
")WITH(" +
" 'connector' = 'hbase-2.2'," +
" 'table-name' = 'default:demo_cdc'," +
" 'zookeeper.quorum' = 'hadoop171:2181,hadoop172:2181,hadoop173:2181'" +
")");
tableEnv.executeSql("insert into hbase_mysql_cdc select id,ROW(username,password) from mysql_cdc");
tableEnv.executeSql("select * from mysql_cdc").print();
env.execute("hbase");
}
}
注意一定要设置主键