Flink入门之Source,包括集合读取、文本读取、元素读取、Kafka读取、自定义Source
学习资料都来源于尚硅谷,开源免费,很感谢他们戳我直达
数据源格式为
sensor_1, 1547718199, 35.80018327300259
sensor_1, 1547718210, 28.39839108328901
sensor_6, 1547718203, 15.40298439340308
sensor_6, 1547718211, 23.76213902108990
sensor_1, 1547718201, 34.12333412589598
sensor_7, 1547718202, 6.720945201171228
sensor_10, 1547718205, 38.1010676048934
sensor_10, 1547718206, 33.9334534654789
更多的细节请看代码注释
老生常谈,CSDN不支持Scala,看官请将就
import java.util.Properties
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011
import scala.util.Random
case class SensorReading(id: String, timestamp: Long, temperature: Double)
// 自定义Source,一般用于测试
class MySensorSource extends SourceFunction[SensorReading] {
// 定义个flag控制运行状态
var running: Boolean = true
// 定义运行次数
var times = 0
override def cancel(): Unit = {
running = false
}
override def run(ctx: SourceContext[SensorReading]): Unit = {
val rand = new Random()
var curTemp = 1.to(4).map(
i => ("sensor_" + i, 23 + rand.nextGaussian() * 10)
)
while (running) {
// 更新温度值
curTemp = curTemp.map(
t => (t._1, t._2 + rand.nextGaussian())
)
// 获取当前时间戳
val curTime = System.currentTimeMillis()
curTemp.foreach(
t => ctx.collect(SensorReading(t._1, curTime, t._2))
)
Thread.sleep(100)
times += 1
// 运行n次后退出
if (times == 2) {
cancel()
}
}
}
}
object readFrom {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(2)
// 一、从集合中读取
val stream = env.fromCollection(List(
SensorReading("Sensor_1", 1547718199, 35.80018327300259),
SensorReading("Sensor_6", 1547718201, 15.402984393403084),
SensorReading("Sensor_7", 1547718202, 6.720945201171228),
SensorReading("Sensor_10", 1547718205, 38.101067604893444)
))
stream.print("Stream: ")
// 二、从文本中读取
val path = "src/main/resources/tep.txt"
val streamText = env.readTextFile(path)
streamText.print("StreamText: ")
// 三、直接读取元素
val streamElements = env.fromElements(1, 2.0, "Haha").print("StreamElements: ")
// 四、读取kafka消息
// 设置配置参数
val properties = new Properties()
properties.setProperty("bootstrap.servers", "192.168.3.60:9092")
properties.setProperty("group.id", "consumer-group")
properties.setProperty("key.deserializer",
"org.apache.kafka.common.serialization.StringDeserializer")
properties.setProperty("value.deserializer",
"org.apache.kafka.common.serialization.StringDeserializer")
properties.setProperty("auto.offset.reset", "latest")
// "sensor"是topic名
val streamKafka = env.addSource(new FlinkKafkaConsumer011[String]("sensor",
new SimpleStringSchema(), properties))
streamKafka.print("StreamKafka: ")
// 五、自定义Source
val streamDefinded = env.addSource(new MySensorSource())
streamDefinded.print("StreamDefinded: ")
// 执行
env.execute()
}
}