package table
import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment}
import org.apache.flink.table.api.bridge.scala.{StreamTableEnvironment}
import org.apache.flink.table.api.{DataTypes, EnvironmentSettings, Table,_}
import org.apache.flink.table.descriptors.{Csv, Kafka, Schema}
object Table01 {
def main(args: Array[String]): Unit = {
// 创建stream环境
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
// 创建streamtable环境
val settings: EnvironmentSettings = EnvironmentSettings
.newInstance()
.useBlinkPlanner()
.inStreamingMode()
.build()
val environment: StreamTableEnvironment = StreamTableEnvironment
.create(env,settings)
// 注册一张表从kafka读取数据
environment.connect(new Kafka()
.version("0.11")
.topic("hadoop")
.property("bootstrap.servers","hadoop01:9092")
.property("zookeeper.connect","hadoop01:2181")
)
.withFormat(new Csv())
.withSchema(new Schema()
.field("id",DataTypes.INT())
.field("name",DataTypes.STRING())
.field("age",DataTypes.DOUBLE())
)
.createTemporaryTable("kafkaInputTable")
// 扫描注册表
val firstTable: Table = environment.from("kafkaInputTable")
print(firstTable.toString)
//查询表
val resultTable: Table = firstTable
.groupBy('id)
.select('id,'id.sum as 'num)
// .filter("id === 1")
resultTable.printSchema()
// sql查询
val sqlTable: Table = environment.sqlQuery(
"""
select
id,name,age
from kafkaInputTable
""".stripMargin)
//注册一张表用于把计算结果输出
environment.connect(new Kafka()
.version("0.11")
.topic("hadoop01")
.property("bootstrap.servers","hadoop01:9092,hadoop02:9092,hadoop03:9092")
.property("zookeeper.connect","hadoop01:2181")
)
.withFormat(new Csv())
.withSchema(new Schema()
.field("id",DataTypes.INT())
.field("name",DataTypes.STRING())
.field("age",DataTypes.DOUBLE())
)
.createTemporaryTable("kafkaOutputTable")
// 将结果发送到注册的 TableSink
// resultTable.insertInto("kafkaOutputTable")
sqlTable.executeInsert("kafkaOutputTable")
}
}
maven引入
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>yansen.com</groupId>
<artifactId>flink</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<flink.version>1.11.0</flink.version>
<scala.version>2.12</scala.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala_${scala.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-common</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka-0.11_${scala.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-scala-bridge_${scala.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge_${scala.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-csv</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner-blink_${scala.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<!-- <dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner_${scala.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_${scala.version}</artifactId>
<version>${flink.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<!--该插件用于将scala代码编译成class文件-->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<goals>
<!--声明绑定到maven的compile阶段-->
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.0.0</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
小试牛刀,欢迎批评指正