**Flink入门Demo:**涉及到的Transform算子有map、flatMap、filter、keyBy、reduce、split、select、connect、union,Sink方法有kafka sink与JDBC sink,其中JDBC sink需要自定义
学习资料都来源于尚硅谷,开源免费,很感谢他们戳我直达
数据源格式为
sensor_1, 1547718199, 35.80018327300259
sensor_1, 1547718210, 28.39839108328901
sensor_6, 1547718203, 15.40298439340308
sensor_6, 1547718211, 23.76213902108990
sensor_1, 1547718201, 34.12333412589598
sensor_7, 1547718202, 6.720945201171228
sensor_10, 1547718205, 38.1010676048934
sensor_10, 1547718206, 33.9334534654789
更多的细节请看代码注释
老生常谈,CSDN不支持Scala,看官请将就
import java.sql._
import com.ck.readFrom.SensorReading
import org.apache.flink.api.common.functions.{RichMapFunction, FilterFunction}
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
import org.apache.flink.streaming.api.functions.sink.SinkFunction.Context
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011
object TransformTest {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
val streamFromFile = env.readTextFile("src/main/resources/tep.txt")
val stream: DataStream[SensorReading] = streamFromFile.map(data => {
val dataArray = data.split(",")
SensorReading(dataArray(0).trim, dataArray(1).trim.toLong, dataArray(2).trim.toDouble)
})
// 相关的聚合方法
val aggStream = stream.keyBy(_.id).reduce((x, y) => SensorReading(x.id, y.timestamp + 1, y.temperature + 10))
// split与select算子,通过两算子配合才能便流彻底切分成两个DataStream
val splitStream = stream.split(data => {
if (data.temperature > 30) Seq("high") else Seq("low")
})
val high = splitStream.select("high")
val low = splitStream.select("low")
val all = splitStream.select("high", "low")
// connect与coMap/coFlatMap,数据类型可以不一样,但只能合并两条流
val warning = high.map(data => (data.id, data.temperature))
val connected = warning.connect(low)
val coMap = connected.map (
warningData => (warningData._1, warningData._2, "warning"),
lowData => (lowData.id, "healthy")
)
// union不能合并类型不同的流,可以合并多个
val unionStream = high.union(low)
// 函数类,自己实现UDF
val udfStream = stream.filter(new MyFilter())
// 匿名函数,注意scala的缩写方法
val lambdaStream = stream.filter(_.temperature > 30)
// 富函数demo
val richStream = stream.map(new MyMapper())
// 打印结果
// aggStream.print("aggStream ")
// high.print("high ")
// low.print("low ")
// all.print("all ")
// coMap.print("coMap ")
// unionStream.print("unionStream ")
// udfStream.print("udfStream ")
// lambdaStream.print("lambdaStream ")
// richStream.print("richStream")
// sink结果到kafka
val kafkaSinkStream: DataStream[String] = streamFromFile.map(data => {
val dataArray = data.split(",")
SensorReading(dataArray(0).trim, dataArray(1).trim.toLong, dataArray(2).trim.toDouble).toString
// 转String方便序列化
})
kafkaSinkStream.addSink(new FlinkKafkaProducer011[String]("localhost:9092", "test", new SimpleStringSchema()))
// sink结果到MySQL
stream.addSink(new myJDBCSink())
// 执行
env.execute("TransformTest")
}
}
class MyFilter() extends FilterFunction[SensorReading] {
override def filter(t: SensorReading): Boolean = {
t.id.startsWith("sensor_1")
}
}
// RichMapFunction参数一个为传入,一个为传出类型
class MyMapper() extends RichMapFunction[SensorReading, String] {
override def map(in: SensorReading): String = {
in.id
}
}
class myJDBCSink() extends RichSinkFunction[SensorReading] {
var conn: Connection = _
var insertStmt: PreparedStatement = _
var updateStmt: PreparedStatement = _
// 初始化,创建连接和预编译语句
override def open(parameters: Configuration): Unit = {
super.open(parameters)
conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/test", "root", "000000")
insertStmt = conn.prepareStatement("INSERT INTO tablename (sensor, temp) VALUES (?, ?)")
updateStmt = conn.prepareStatement("UPDATE temperatures SET temp = ? WHERE sensor = ?")
}
// 调用连接执行SQL
override def invoke(value: SensorReading, context: Context[_]): Unit = {
// 执行更新语句
updateStmt.setDouble(1, value.temperature)
updateStmt.setString(2, value.id)
updateStmt.execute()
// 如果updataStmt没查到执行插入
if (updateStmt.getUpdateCount == 0) {
insertStmt.setString(1, value.id)
insertStmt.setDouble(2, value.temperature)
insertStmt.execute()
}
}
// 关闭时清理工作
override def close(): Unit = {
insertStmt.close()
updateStmt.close()
conn.close()
}
}