flink-入门功能整合(udf,创建临时表table,使用flink sql)

说明

• 本次测试用scala,java版本大体都差不多,不再写两个版本了
• StreamTableEnvironment做了很多调整,目前很多网上的样例使用的都是过时的api,本次代码测试使用的都是官方doc中推荐使用的新api
• 本次测试代码主要测试了三个基本功能:1.UDF 2.流处理Table的创建以及注册 3.Flink Sql
• 依赖有很多是多余的,懒得删了

依赖

<dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-scala_2.12</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_2.12</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.12</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.12</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-core</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-test-utils-junit</artifactId>
            <version>1.12.1</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table</artifactId>
            <version>1.12.1</version>
            <type>pom</type>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-common</artifactId>
            <version>1.12.1</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-scala_2.12</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_2.12</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-scala-bridge_2.12</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_2.12</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_2.12</artifactId>
            <version>1.12.1</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-runtime-blink_2.12</artifactId>
            <version>1.12.1</version>
            <scope>provided</scope>
        </dependency>

    </dependencies>

代码

import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import org.apache.flink.table.api.EnvironmentSettings
import org.apache.flink.table.api.Expressions.$
import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment
import org.apache.flink.table.functions.ScalarFunction

/**
 * word count
 * 1.create and use udf function
 * 2.create tmp table
 * 3.use table sql
 *
 * @author:sssuperMario
 * @date:2021-03-02
 */
object WordCount {
  def main(args: Array[String]): Unit = {
    //create stream runtime environment
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    //with blink planner setting
    val settings = EnvironmentSettings.newInstance().useBlinkPlanner().build()
    //create table environment
    val tableEnv = StreamTableEnvironment.create(env, settings)
    //register udf
    registerUDF(tableEnv)
    //input datasource
    val inputDataStream = env.socketTextStream("localhost", 8888)
    //word count
    val ds: DataStream[(String, Int)] = inputDataStream.filter(_.nonEmpty)
      .flatMap(_.split(" "))
      .map((_, 1))
      .keyBy(_._1)
      .sum(1)
    //register table
    tableEnv.createTemporaryView("wordcount_tmp", ds, $("word"), $("grand_total"))
    //execute sql and print result
    tableEnv.executeSql("select flink_udf_test(word) word,grand_total from wordcount_tmp")
      .print()
    //execute
    env.execute("WordCount")
  }

  /**
   * register udf
   *
   * @param env
   */
  def registerUDF(env: StreamTableEnvironment): Unit = {
    env.createTemporaryFunction("flink_udf_test", new TestUDF)
  }

  /**
   * udf
   */
  class TestUDF extends ScalarFunction {

    def eval(s: String): String = {
      "<" + s + ">"
    }
  }

}

终端启动端口

nc -lk -p 8888

控制台输出

+----+--------------------------------+-------------+
| op |                           word | grand_total |
+----+--------------------------------+-------------+
| +I |                            <a> |           1 |
| +I |                            <d> |           1 |
| +I |                            <a> |           2 |
| +I |                            <d> |           2 |