1、默认是500Lms一次checkpoint,watermark周期性生成,默认是200ms
2、失败重启策略
3、如果连续两次温度差值超过10,则报警
package flinkSourse
import akka.stream.TLSClientAuth.None
import org.apache.flink.api.common.functions.RichFlatMapFunction
import org.apache.flink.api.common.restartstrategy.RestartStrategies
import org.apache.flink.api.common.restartstrategy.RestartStrategies.RestartStrategyConfiguration
import org.apache.flink.api.common.state.{ListState, ValueState, ValueStateDescriptor}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector
import org.apache.flink.api.common.time
import org.apache.flink.api.common.time.Time
//flink中一些简单的算子有些不需要状态的保存,但聚合算子、min或sum等算子、窗口等算子都需要state保存 ,flink会进行状态一致性、故障处理、高效的存储和访问等功能
//operator State 状态的作用范围是当前任务算子,不同的task间不能访问(需要序列化传输的),当前任务的所处理的所有数据都可以访问到该状态 。keyed state的作用范围是当前key的聚合或者窗口算子
object FlinkState {
def main(args: Array[String]): Unit = {
val executionEnvironment: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
executionEnvironment.setParallelism(1)
executionEnvironment.enableCheckpointing(1000L); //默认是500Lms一次checkpoint,watermark周期性生成,默认是200ms
executionEnvironment.getCheckpointConfig.setCheckpointTimeout(60000) //从source产生checkpoint到sink的checkpoint完成的时间最多60s,否则直接丢弃
//默认是1个,多个checkpoint并行执行,防止背压的情况,// 当一个checkpoint任务的时候,如果当前checkpoint任务1s内没处理完,后一个checkpint的barier的产生会延后
executionEnvironment.getCheckpointConfig.setMaxConcurrentCheckpoints(2)
//失败率重启策略在失败后重启工作,当超过配置的失败次数(3次)时工作宣告失败
executionEnvironment.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 10000)) //重启的时间间隔为10s
//失败率重启策略在失败后重启工作,当超过 failure rate一个时间段内(5 min)的失败次数(3次)时工作宣告失败。 在连续两次重启尝试中,该重启策略会等待一端固定的时间。
executionEnvironment.setRestartStrategy(RestartStrategies.failureRateRestart(3, Time.minutes(5), Time.seconds(10)))
val stream2: DataStream[String] = executionEnvironment.socketTextStream("127.0.0.1", 1111)
val transforStream: DataStream[SensorReading] = stream2.map(data => {
val tmpList: Array[String] = data.split(",")
SensorReading(tmpList(0), tmpList(1).toLong, tmpList(2).toDouble)
})
//如果连续两次温度差值超过10,则报警
val alterStream: DataStream[(String, Double, Double)] = transforStream.keyBy(_.id).flatMap(new TempChangeAlert(10))
alterStream.print()
executionEnvironment.execute("state")
}
}
class TempChangeAlert(threshold: Double) extends RichFlatMapFunction[SensorReading, (String, Double, Double)] {
var lastValueState: ValueState[Double] = _
var listState: ListState[Double] = _
var is_first: Boolean = true
override def open(parameters: Configuration): Unit = {
lastValueState = getRuntimeContext.getState(new ValueStateDescriptor[Double]("tempState", classOf[Double]))
}
override def flatMap(in: SensorReading, collector: Collector[(String, Double, Double)]): Unit = {
val lastTemp: Double = lastValueState.value()
if ((lastTemp - in.temperature).abs > threshold) {
if (is_first != true) {
collector.collect(in.id, lastTemp, in.temperature)
} else {
is_first = false
}
}
//更新状态
lastValueState.update(in.temperature)
}
}