Flink transform算子详解之一

• 转换算子map、filter、flatMap、keyBy、sum
• flink为了每个算子提供函数和函数类两种传参数形式
• 另外函数类分为Function、RichFunction，后一种可以重写open方法实现数据库查询等

package com.transform

import java.sql.{Connection, DriverManager, PreparedStatement}

import org.apache.flink.api.common.functions.{FilterFunction, FlatMapFunction, RichFlatMapFunction}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector
object One {
  case class Count(w:String, count:Int)
  def main(args: Array[String]): Unit = {
    // 1、获取流式环境变量
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    // 2、普通算子map、filter、flatMap、keyBy、sum用法演示
    val dataStream = env.fromElements("1 2 3 4 5 6 0")
      // 一进多出
      .flatMap(new FlatMapFun)
      // 一进一出，map为样例类
      .map(
        x=>{
          Count(w = x,count = 1)
        }
      )
      // 过滤 true的输出，false的忽略
      .filter(new FilterFun)
      // 分流
      .keyBy(_.w)
      //第二个位置求和
      .sum(1)
    // 3、打印数据
    dataStream.print()
    // 4、执行任务
    env.execute("transform Job")
  }

  class FlatMapFun extends RichFlatMapFunction[String, String] {

    val uri = "jdbc:mysql://jeff200:3306/test_db?characterEncoding=utf-8"
    val user = "root"
    val pwd = "root"
    classOf[com.mysql.jdbc.Driver]
    var conn: Connection= _
    var preStatement: PreparedStatement = _
    override def open(parameters: Configuration): Unit = {
      conn = DriverManager.getConnection(uri, user, pwd)
      val sql =
        """
          |SELECT uid, name
          |FROM t_user
          |WHERE uid = ?
          |""".stripMargin
      preStatement = conn.prepareStatement(sql)
    }

    override def flatMap(t: String, collector: Collector[String]): Unit = {
      val arr = t.split(" ")
      for(str <- arr){
        val uid = str.toInt
        preStatement.setInt(1, uid)
        val result = preStatement.executeQuery()
        // 取出user表包含uid为1的用户
        while (result.next){
          val name = result.getString(2)
          //一个个发射出去
          collector.collect(
            name
          )
        }
      }
    }

    override def close(): Unit = {
      if (conn != null) conn.close
    }
  }

  class FilterFun extends FilterFunction[Count] {
    override def filter(t: Count): Boolean = t.w != "0"
  }
}

• 数据库访问依赖

<!-- mysql 连接器 -->
<dependency>
   <groupId>mysql</groupId>
   <artifactId>mysql-connector-java</artifactId>
   <version>5.1.35</version>
</dependency>

• 运行结果