Flink_状态管理

最新推荐文章于 2022-08-31 17:51:23 发布

weixin_43003792

最新推荐文章于 2022-08-31 17:51:23 发布

阅读量99

点赞数

分类专栏： Flink

本文链接：https://blog.csdn.net/weixin_43003792/article/details/113842394

版权

Flink 专栏收录该内容

26 篇文章 0 订阅

订阅专栏

Flink状态
算子状态Opertaion
键控状态KeyedState ***********
状态后端

一：状态就是本地变量（分布式系统）：Flink中的状态：为了得到最新输出（reduce |window）依赖另外的数据。这些数据就叫做状态。本地变量。
二：状态的类型
1.算子状态（Operator State）当前算子任务都能访问到
a:List state 将状态表示为一组数据的列表
Union State
c:广播状态 broadcast state`

	2.键控状态（Keyed State） 不同的key访问不同状态 更常用。 key by 之后。
			键控状态 对每一个key都保存了自己的状态
			       value state
			       list state`
			  	Map state
			  	Reducing state & Aggregating State 将状态表示为一个用于聚合操作的列表

三：状态编程
//状态必须用RichFunction
class MyRichMapper extends RichMapFunction[SensorReading,String]{
var valueState: ValueState[Double]=_
lazy val listState: ListState[Int] = getRuntimeContext.getListState(new ListStateDescriptor[Int](“liststate”, classOf[Int]))
lazy val mapState: MapState[String, Double] = getRuntimeContext.getMapState(new MapStateDescriptorString,Double)
lazy val reduceState: ReducingState[SensorReading] = getRuntimeContext.getReducingState(new ReducingStateDescriptor[SensorReading](“reducestate”,new MyReduce,classOf[SensorReading]))
override def open(parameters: Configuration): Unit = {
//获取状态直接得到value state
valueState= getRuntimeContext.getState(new ValueStateDescriptorDouble)
}
override def map(value: SensorReading): String = {
//使用valuestate
val myValue: Double = valueState.value()
val list = new util.ArrayListInt
listState.addAll(list)
mapState.contains(“sensor_1”)
val vl: Double = mapState.get(“sensor_1”)
mapState.put(“sensor_1”,value.temperature)
val lists: lang.Iterable[Int] = listState.get()
val sensoreading: SensorReading = reduceState.get()
reduceState.add(value)
//状态读写操作
valueState.update(value.temperature)
value.id
}
}
四：案例要求传感器如果跳变超过10度输出报警信息。温度跳变
Bug :第一次温度未做判断
package com.atguigu.state

import java.{lang, util}

import com.atguigu.sourceandsink.SensorReading
import com.atguigu.window.MyReduce
import org.apache.flink.api.common.functions.{RichFlatMapFunction, RichMapFunction}
import org.apache.flink.api.common.state._
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

object StateTest {
def main(args: Array[String]): Unit = {
//设置时间语义默认
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
env.getConfig.setAutoWatermarkInterval(50)
val inputStream: DataStream[String] = env.socketTextStream(“localhost”,7777)

//提取eventtime
val dataStream: DataStream[SensorReading] = inputStream.map(data => {
  val strings: Array[String] = data.split(",")
  SensorReading(strings(0), strings(1).toLong, strings(2).toDouble)
})
val alertStream: DataStream[(String, Double, Double)] = dataStream
  .keyBy(_.id)
  .flatMap(new TempChangeAlert(10.0))
alertStream.print("alertStream")
//状态编程

env.execute("state test")

}
}
//实现自定义RichFlatmapFunction
class TempChangeAlert(threshold:Double) extends RichFlatMapFunction[SensorReading,(String,Double,Double)]{
//定义状态
lazy val lastTempState: ValueState[Double] = getRuntimeContext.getState(new ValueStateDescriptorDouble)

//每一个数据都调用这里

override def flatMap(value: SensorReading, out: Collector[(String, Double, Double)]): Unit = {
//flatmap用collect输出
val lastTemp: Double = lastTempState.value()
val diff: Double = (value.temperature-lastTemp).abs
if(diff>threshold){ //如果温差大于10度输出温度
out.collect((value.id,lastTemp,value.temperature))
}
//更新状态
lastTempState.update(value.temperature)
}

}
//状态必须用RichFunction
class MyRichMapper extends RichMapFunction[SensorReading,String]{
var valueState: ValueState[Double]=_
lazy val listState: ListState[Int] = getRuntimeContext.getListState(new ListStateDescriptor[Int](“liststate”, classOf[Int]))
lazy val mapState: MapState[String, Double] = getRuntimeContext.getMapState(new MapStateDescriptorString,Double)
lazy val reduceState: ReducingState[SensorReading] = getRuntimeContext.getReducingState(new ReducingStateDescriptor[SensorReading](“reducestate”,new MyReduce,classOf[SensorReading]))
override def open(parameters: Configuration): Unit = {
//获取状态直接得到value state
valueState= getRuntimeContext.getState(new ValueStateDescriptorDouble)
}
override def map(value: SensorReading): String = {
//使用valuestate
val myValue: Double = valueState.value()
val list = new util.ArrayListInt
listState.addAll(list)
mapState.contains(“sensor_1”)
val vl: Double = mapState.get(“sensor_1”)
mapState.put(“sensor_1”,value.temperature)
val lists: lang.Iterable[Int] = listState.get()
val sensoreading: SensorReading = reduceState.get()
reduceState.add(value)
//状态读写操作
valueState.update(value.temperature)
value.id
}
}

//简单形式可以避免首次输出
val alerStream: DataStream[(String, Double, Double)] = dataStream
.keyBy(_.id)
//.flatMap(new TempChangeAlert(10.0))
.flatMapWithState[(String, Double, Double), Double] {
case (data: SensorReading, None) => (List.empty, Some(data.temperature))
case (data: SensorReading, lasTemp: Some[Double]) => {
val diff: Double = (data.temperature - lasTemp.get).abs
if (diff > 10.0) { //如果温差大于10度输出温度
(List((data.id, lasTemp.get, data.temperature)), Some(data.temperature))
}
else (List.empty, Some(data.temperature))
}
}
package com.atguigu.state

import java.{lang, util}

//提取eventtime
val dataStream: DataStream[SensorReading] = inputStream.map(data => {
  val strings: Array[String] = data.split(",")
  SensorReading(strings(0), strings(1).toLong, strings(2).toDouble)
})
val alerStream: DataStream[(String, Double, Double)] = dataStream
  .keyBy(_.id)
  //.flatMap(new TempChangeAlert(10.0))
  .flatMapWithState[(String, Double, Double), Double] {
  case (data: SensorReading, None) => (List.empty, Some(data.temperature))
  case (data: SensorReading, lasTemp: Some[Double]) => {
    val diff: Double = (data.temperature - lasTemp.get).abs
    if (diff > 10.0) { //如果温差大于10度 输出温度
      (List((data.id, lasTemp.get, data.temperature)), Some(data.temperature))
    }
    else (List.empty, Some(data.temperature))
  }
}
alerStream.print("alertStream")
//状态编程

env.execute("state test")

//每一个数据都调用这里

override def map(value: SensorReading): String = {
//使用valuestate
val myValue: Double = valueState.value()
val list = new util.ArrayListInt
listState.addAll(list)
mapState.contains(“sensor_1”)
val vl: Double = mapState.get(“sensor_1”)
mapState.put(“sensor_1”,value.temperature)
val lists: lang.Iterable[Int] = listState.get()
val sensoreading: SensorReading = reduceState.get()
reduceState.add(value)
//状态读写操作
valueState.update(value.temperature)
value.id
}
}

weixin_43003792

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
Flink_状态管理

一：本地变量（分布式系统）：Flink中的状态：为了得到最新输出（reduce |window）依赖另外的数据。这些数据就叫做状态。本地变量。二：状态的类型1.算子状态（Operator State）当前算子任务都能访问到a:List state 将状态表示为一组数据的列表b:Union state 将c:广播状态 broadcast state` 2.键控状态（Keyed State）不同的key访问不同状态更常用。 key by 之后。键控状态对每一个key都保存了自己的
复制链接

扫一扫