- Flink CDC 读取kafka json格式数据
package com.com.cdc
import org.apache.flink.streaming.api.scala._
import org.apache.flink.table.api.{EnvironmentSettings, Table}
import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment
import org.apache.flink.types.Row
object KafkaJson {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val bsSettings = EnvironmentSettings.newInstance.useBlinkPlanner.inStreamingMode.build
val sTableEnv = StreamTableEnvironment.create(env, bsSettings)
val kfkSql =
s"""
|CREATE TABLE kafka_table (
| common ROW(uid string),
| ts BIGINT
|) WITH (
| 'connector' = 'kafka',
| 'topic' = 'test',
| 'properties.bootstrap.servers' = 'jeff200:9092',
| 'format' = 'json',
| 'scan.startup.mode' = 'earliest-offset'
|)
|""".stripMargin
sTableEnv.executeSql(kfkSql)
val filterSql =
s"""
|SELECT uid, ts
|FROM kafka_table
|WHERE uid > 0
""".stripMargin
val table: Table = sTableEnv.sqlQuery(filterSql)
val kafkaStream:DataStream[Row] = sTableEnv.toAppendStream(table)
kafkaStream.print()
env.execute("kafka cdc")
}
}
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-json</artifactId>
<version>${flink.version}</version>
<scope>${scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-scala-bridge_2.11</artifactId>
<version>${flink.version}</version>
<scope>${scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner-blink_2.11</artifactId>
<version>${flink.version}</version>
<scope>${scope}</scope>
</dependency>
{"ts": 1607003021, "common": {"uid": 1}, "displays": [{"item": 1, "type": "goods"},{"item": 2, "type": "goods"}]}