[Scala] Flink Source Demo

Flink入门之Source，包括集合读取、文本读取、元素读取、Kafka读取、自定义Source

学习资料都来源于尚硅谷，开源免费，很感谢他们戳我直达

数据源格式为

sensor_1, 1547718199, 35.80018327300259
sensor_1, 1547718210, 28.39839108328901
sensor_6, 1547718203, 15.40298439340308
sensor_6, 1547718211, 23.76213902108990
sensor_1, 1547718201, 34.12333412589598
sensor_7, 1547718202, 6.720945201171228
sensor_10, 1547718205, 38.1010676048934
sensor_10, 1547718206, 33.9334534654789

更多的细节请看代码注释
老生常谈，CSDN不支持Scala，看官请将就

import java.util.Properties
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011

import scala.util.Random

case class SensorReading(id: String, timestamp: Long, temperature: Double)

// 自定义Source，一般用于测试
class MySensorSource extends SourceFunction[SensorReading] {
  // 定义个flag控制运行状态
  var running: Boolean = true
  // 定义运行次数
  var times = 0

  override def cancel(): Unit = {
    running = false
  }

  override def run(ctx: SourceContext[SensorReading]): Unit = {
    val rand = new Random()
    var curTemp = 1.to(4).map(
      i => ("sensor_" + i, 23 + rand.nextGaussian() * 10)
    )

    while (running) {
      // 更新温度值
      curTemp = curTemp.map(
        t => (t._1, t._2 + rand.nextGaussian())
      )
      // 获取当前时间戳
      val curTime = System.currentTimeMillis()

      curTemp.foreach(
        t => ctx.collect(SensorReading(t._1, curTime, t._2))
      )
      Thread.sleep(100)
      times += 1
      // 运行n次后退出
      if (times == 2) {
        cancel()
      }
    }
  }
}

object readFrom {
  def main(args: Array[String]): Unit =  {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(2)
    // 一、从集合中读取
    val stream = env.fromCollection(List(
        SensorReading("Sensor_1", 1547718199, 35.80018327300259),
        SensorReading("Sensor_6", 1547718201, 15.402984393403084),
        SensorReading("Sensor_7", 1547718202, 6.720945201171228),
        SensorReading("Sensor_10", 1547718205, 38.101067604893444)
      ))
    stream.print("Stream: ")

    // 二、从文本中读取
    val path = "src/main/resources/tep.txt"
    val streamText = env.readTextFile(path)
    streamText.print("StreamText: ")

    // 三、直接读取元素
    val streamElements = env.fromElements(1, 2.0, "Haha").print("StreamElements: ")

    // 四、读取kafka消息
    // 设置配置参数
    val properties = new Properties()
    properties.setProperty("bootstrap.servers", "192.168.3.60:9092")
    properties.setProperty("group.id", "consumer-group")
    properties.setProperty("key.deserializer",
      "org.apache.kafka.common.serialization.StringDeserializer")
    properties.setProperty("value.deserializer",
      "org.apache.kafka.common.serialization.StringDeserializer")
    properties.setProperty("auto.offset.reset", "latest")
    // "sensor"是topic名
    val streamKafka = env.addSource(new FlinkKafkaConsumer011[String]("sensor",
      new SimpleStringSchema(), properties))
    streamKafka.print("StreamKafka: ")

    // 五、自定义Source
    val streamDefinded = env.addSource(new MySensorSource())
    streamDefinded.print("StreamDefinded: ")


    // 执行
    env.execute()
  }
}