Flink_基于KeyedProcessFunction的使用案例

package com.atguigu.processfunction

import java.util.concurrent.TimeUnit

import com.atguigu.sourceandsink.SensorReading
import org.apache.flink.api.common.restartstrategy.RestartStrategies
import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.api.common.time.Time
import org.apache.flink.configuration.Configuration
import org.apache.flink.contrib.streaming.state.RocksDBStateBackend
import org.apache.flink.streaming.api.{CheckpointingMode, TimeCharacteristic}
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

object TestProcessKeyFunction {
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
val inputStream: DataStream[String] = env.socketTextStream(“localhost”,7777)
env.enableCheckpointing(1000)//打开检查点保存 默认excetly once 保存的时间间隔
env.getCheckpointConfig.setCheckpointingMode(CheckpointingMode.AT_LEAST_ONCE) //调低jibie
env.getCheckpointConfig.setCheckpointTimeout(60000) //超时时间
env.getCheckpointConfig.setMaxConcurrentCheckpoints(2) //允许同时并行的最大Checkpoint
env.getCheckpointConfig.setMinPauseBetweenCheckpoints(1000) //两次保存时至少的时间间隔
env.getCheckpointConfig.setPreferCheckpointForRecovery(false) //false下与手动保存哪个近用哪个
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3,10000L))
env.setRestartStrategy(RestartStrategies.failureRateRestart(5,Time.of(5,TimeUnit.MINUTES),Time.of(10,TimeUnit.SECONDS)))
//提取eventtime
val dataStream: DataStream[SensorReading] = inputStream.map(data => {
val strings: Array[String] = data.split(",")
SensorReading(strings(0), strings(1).toLong, strings(2).toDouble)
})
val keyStream: KeyedStream[SensorReading, String] = dataStream.keyBy(_.id)
val result: DataStream[String] = keyStream.process(new TempIncreWarning(10000L))
result.print(“process”)
env.execute(“myProcessTest”)
}
}
//实现自定义keyedProcessFunction
class TempIncreWarning(interval: Long) extends KeyedProcessFunction[String,SensorReading,String]{
//判断温度连续上升
//定义状态 保存上一个温度值进行比较 保存注册定时器的时间
lazy val lastTempState: ValueState[Double] = getRuntimeContext.getState(new ValueStateDescriptorDouble)
lazy val timerTsState: ValueState[Long] = getRuntimeContext.getState(new ValueStateDescriptorLong)
//每一条数据都会调到该方法
override def processElement(value: SensorReading, ctx: KeyedProcessFunction[String, SensorReading, String]#Context, out: Collector[String]): Unit = {
val lastTemp: Double = lastTempState.value() //上次温度值
val timerTs: Long = timerTsState.value()
//
lastTempState.update(value.temperature)
//当前温度值和上次温度进行比较
if(value.temperature>lastTemp && timerTs == 0){ //如果本次温度大于上一次温度并且没有定时器，那么注册当前时间戳+10s为定时器
val ts: Long = ctx.timerService().currentProcessingTime()+interval
ctx.timerService().registerProcessingTimeTimer(ts) //注册定时器
timerTsState.update(ts) //将当前定时器更新到状态里面
}else if (value.temperature<lastTemp){
//如果温度下降 删除定时器
ctx.timerService().deleteProcessingTimeTimer(timerTs)
timerTsState.clear() //清空状态
}
}
//触发时要做的操作
override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, SensorReading, String]#OnTimerContext, out: Collector[String]): Unit = {
out.collect(“传感器”+ctx.getCurrentKey+“的温度连续”+interval/1000+“上升”)
timerTsState.clear()
}
}
class MyKeyedProceddFunction extends KeyedProcessFunction[String,SensorReading,SensorReading]{
var myState:ValueState[Int] = _
override def open(parameters: Configuration): Unit = super.open(parameters)

override def processElement(value: SensorReading, ctx: KeyedProcessFunction[String, SensorReading, SensorReading]#Context, out: Collector[SensorReading]): Unit = {
ctx.getCurrentKey
ctx.timestamp()
//定义定时器 按照事件时间
ctx.timerService().registerEventTimeTimer(ctx.timestamp()+600000L)
}

override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, SensorReading, SensorReading]#OnTimerContext, out: Collector[SensorReading]): Unit = super.onTimer(timestamp, ctx, out)
}