Flink_ProcessFunctionAPI

一：作用：获取eventtime watermark
processFunction 继承RichFunction(生命周期，获取运行时上下文，状态编程) 独特含有定时器 onTemer()
注册未来时刻定时器
分流操作：
 ProcessFunction //最底层 dataStream.process()
 KeyedProcessFunction
 CoProcessFunction
 ProcessJoinFunction
 BroadcastProcessFunction
 KeyedBroadcastProcessFunction
 ProcessWindowFunction
 ProcessAllWindowFunction
二： 需求 10s内如果传感器连续温度上升输出报警信息 定时器
package com.atguigu.processfunction

import com.atguigu.sourceandsink.SensorReading
import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

object TestProcessKeyFunction {
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
env.getConfig.setAutoWatermarkInterval(50)
val inputStream: DataStream[String] = env.socketTextStream(“localhost”,7777)

//提取eventtime
val dataStream: DataStream[SensorReading] = inputStream.map(data => {
  val strings: Array[String] = data.split(",")
  SensorReading(strings(0), strings(1).toLong, strings(2).toDouble)
})
val keyStream: KeyedStream[SensorReading, String] = dataStream.keyBy(_.id)
val result: DataStream[String] = keyStream.process(new TempIncreWarning(10000L))
result.print("process")
env.execute("myProcessTest")

}
}
//实现自定义keyedProcessFunction
class TempIncreWarning(interval: Long) extends KeyedProcessFunction[String,SensorReading,String]{
//判断温度连续上升
//定义状态 保存上一个温度值进行比较 保存注册定时器的时间
lazy val lastTempState: ValueState[Double] = getRuntimeContext.getState(new ValueStateDescriptorDouble)
lazy val timerTsState: ValueState[Long] = getRuntimeContext.getState(new ValueStateDescriptorLong)
//每一条数据都会调到该方法
override def processElement(value: SensorReading, ctx: KeyedProcessFunction[String, SensorReading, String]#Context, out: Collector[String]): Unit = {
val lastTemp: Double = lastTempState.value() //上次温度值
val timerTs: Long = timerTsState.value()
//
lastTempState.update(value.temperature)
//当前温度值和上次温度进行比较
if(value.temperature>lastTemp && timerTs == 0){ //如果本次温度大于上一次温度并且没有定时器，那么注册当前时间戳+10s为定时器
val ts: Long = ctx.timerService().currentProcessingTime()+interval
ctx.timerService().registerProcessingTimeTimer(ts) //注册定时器
timerTsState.update(ts) //将当前定时器更新到状态里面
}else if (value.temperature<lastTemp){
//如果温度下降 删除定时器
ctx.timerService().deleteProcessingTimeTimer(timerTs)
timerTsState.clear() //清空状态
}
}
//触发时要做的操作
override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, SensorReading, String]#OnTimerContext, out: Collector[String]): Unit = {
out.collect(“传感器”+ctx.getCurrentKey+“的温度连续”+interval/1000+“上升”)
timerTsState.clear()
}
}
class MyKeyedProceddFunction extends KeyedProcessFunction[String,SensorReading,SensorReading]{
var myState:ValueState[Int] = _
override def open(parameters: Configuration): Unit = super.open(parameters)

override def processElement(value: SensorReading, ctx: KeyedProcessFunction[String, SensorReading, SensorReading]#Context, out: Collector[SensorReading]): Unit = {
ctx.getCurrentKey
ctx.timestamp()
//定义定时器 按照事件时间
ctx.timerService().registerEventTimeTimer(ctx.timestamp()+600000L)
}

override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, SensorReading, SensorReading]#OnTimerContext, out: Collector[SensorReading]): Unit = super.onTimer(timestamp, ctx, out)
}
三：获取侧输出流 按照温度大于30度时分流到一个新流。
package com.atguigu.processfunction

import com.atguigu.sourceandsink.SensorReading
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

object SideOutputTest {
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
env.getConfig.setAutoWatermarkInterval(50)
val inputStream: DataStream[String] = env.socketTextStream(“localhost”,7777)

//提取eventtime
val dataStream: DataStream[SensorReading] = inputStream.map(data => {
  val strings: Array[String] = data.split(",")
  SensorReading(strings(0), strings(1).toLong, strings(2).toDouble)
})
val result: DataStream[SensorReading] = dataStream.process(new SplitTempProcess(30.0))
val lowDataStream: DataStream[(String, Long, Double)] = result.getSideOutput(new OutputTag[(String,Long,Double)]("low"))
result.print("result")
lowDataStream.print("lowdatastream")
env.execute("Split Out Test")

}
}
//实现自定义ProcessFuncetion
class SplitTempProcess(threshold:Double) extends ProcessFunction[SensorReading,SensorReading]{
override def processElement(value: SensorReading, ctx: ProcessFunction[SensorReading, SensorReading]#Context, out: Collector[SensorReading]): Unit = {
if(value.temperature > threshold){ //如果当前温度值大于30度 输出到主流
out.collect(value)
}else{ //如果不超过30度 输出到侧输出流
ctx.output(new OutputTag(String,Long,Double),(value.id,value.timestamp,value.temperature))
}
}
}