State-理解原理即可
Flink中状态的自动管理
之前写的Flink代码中其实已经做好了状态自动管理,如
发送hello ,得出(hello,1)
再发送hello ,得出(hello,2)
说明Flink已经自动的将当前数据和历史状态/历史结果进行了聚合,做到了状态的自动管理
在实际开发中绝大多数情况下,我们直接使用自动管理即可
一些特殊情况才会使用手动的状态管理!—后面项目中会使用!
所以这里得先学习state状态如何手动管理!
package cn.itcast.source;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
/**
* Author itcast
* Desc 演示DataStream-Source-基于Socket
*/
public class SourceDemo03_Socket {
public static void main(String[] args) throws Exception {
//TODO 0.env
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
//TODO 1.source
DataStream<String> lines = env.socketTextStream("node1", 9999);
//TODO 2.transformation
/*SingleOutputStreamOperator<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
@Override
public void flatMap(String value, Collector<String> out) throws Exception {
String[] arr = value.split(" ");
for (String word : arr) {
out.collect(word);
}
}
});
words.map(new MapFunction<String, Tuple2<String,Integer>>() {
@Override
public Tuple2<String, Integer> map(String value) throws Exception {
return Tuple2.of(value,1);
}
});*/
//注意:下面的操作将上面的2步合成了1步,直接切割单词并记为1返回
SingleOutputStreamOperator<Tuple2<String, Integer>> wordAndOne = lines.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
@Override
public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
String[] arr = value.split(" ");
for (String word : arr) {
out.collect(Tuple2.of(word, 1));
}
}
});
SingleOutputStreamOperator<Tuple2<String, Integer>> result = wordAndOne.keyBy(t -> t.f0).sum(1);
//TODO 3.sink
result.print();
//TODO 4.execute
env.execute();
}
}
无状态计算和有状态计算
- 无状态计算,不需要考虑历史值, 如map
hello --> (hello,1)
hello --> (hello,1)
[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-56Sbptxv-1624947045215)(Flink-day03.assets/1610871016671.png)]
- 有状态计算,需要考虑历史值,如:sum
hello , (hello,1)
hello , (hello,2)
状态分类
- State
- ManagerState–开发中推荐使用 : Fink自动管理/优化,支持多种数据结构
- KeyState–只能在keyedStream上使用,支持多种数据结构
- OperatorState–一般用在Source上,支持ListState
- KeyState–只能在keyedStream上使用,支持多种数据结构
- RawState–完全由用户自己管理,只支持byte[],只能在自定义Operator上使用
- OperatorState
- ManagerState–开发中推荐使用 : Fink自动管理/优化,支持多种数据结构
分类详细图解:
代码演示-ManagerState-keyState
https://ci.apache.org/projects/flink/flink-docs-release-1.12/dev/stream/state/state.html
package cn.itcast.state;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
/**
* Author itcast
* Desc 使用KeyState中的ValueState获取流数据中的最大值/实际中可以使用maxBy即可
*/
public class StateDemo01_KeyState {
public static void main(String[] args) throws Exception {
//TODO 0.env
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
//TODO 1.source
DataStream<Tuple2<String, Long>> tupleDS = env.fromElements(
Tuple2.of("北京", 1L),
Tuple2.of("上海", 2L),
Tuple2.of("北京", 6L),
Tuple2.of("上海", 8L),
Tuple2.of("北京", 3L),
Tuple2.of("上海", 4L)
);
//TODO 2.transformation
//需求:求各个城市的value最大值
//实际中使用maxBy即可
DataStream<Tuple2<String, Long>> result1 = tupleDS.keyBy(t -> t.f0).maxBy(1);
//学习时可以使用KeyState中的ValueState来实现maxBy的底层
DataStream<Tuple3<String, Long, Long>> result2 = tupleDS.keyBy(t -> t.f0).map(new RichMapFunction<Tuple2<String, Long>, Tuple3<String, Long, Long>>() {
//-1.定义一个状态用来存放最大值
private ValueState<Long> maxValueState;
//-2.状态初始化
@Override
public void open(Configuration parameters) throws Exception {
//创建状态描述器
ValueStateDescriptor stateDescriptor = new ValueStateDescriptor("maxValueState", Long.class);
//根据状态描述器获取/初始化状态
maxValueState = getRuntimeContext().getState(stateDescriptor);
}
//-3.使用状态
@Override
public Tuple3<String, Long, Long> map(Tuple2<String, Long> value) throws Exception {
Long currentValue = value.f1;
//获取状态
Long historyValue = maxValueState.value();
//判断状态
if (historyValue == null || currentValue > historyValue) {
historyValue = currentValue;
//更新状态
maxValueState.update(historyValue);
return Tuple3.of(value.f0, currentValue, historyValue);
} else {
return Tuple3.of(value.f0, currentValue, historyValue);
}
}
});
//TODO 3.sink
//result1.print();
//4> (北京,6)
//1> (上海,8)
result2.print();
//1> (上海,xxx,8)
//4> (北京,xxx,6)
//TODO 4.execute
env.execute();
}
}
代码演示-ManagerState-OperatorState
https://ci.apache.org/projects/flink/flink-docs-release-1.12/dev/stream/state/state.html
package cn.itcast.state;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;
import org.apache.flink.runtime.state.FunctionInitializationContext;
import org.apache.flink.runtime.state.FunctionSnapshotContext;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import java.util.Iterator;
/**
* Author itcast
* Desc 使用OperatorState中的ListState模拟KafkaSource进行offset维护
*/
public class StateDemo02_OperatorState {
public static void main(String[] args) throws Exception {
//TODO 0.env
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
env.setParallelism(1);//并行度设置为1方便观察
//下面的Checkpoint和重启策略配置先直接使用,下次课学
env.enableCheckpointing(1000);//每隔1s执行一次Checkpoint
env.setStateBackend(new FsStateBackend("file:///D:/ckp"));
env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
//固定延迟重启策略: 程序出现异常的时候,重启2次,每次延迟3秒钟重启,超过2次,程序退出
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(2, 3000));
//TODO 1.source
DataStreamSource<String> ds = env.addSource(new MyKafkaSource()).setParallelism(1);
//TODO 2.transformation
//TODO 3.sink
ds.print();
//TODO 4.execute
env.execute();
}
//使用OperatorState中的ListState模拟KafkaSource进行offset维护
public static class MyKafkaSource extends RichParallelSourceFunction<String> implements CheckpointedFunction {
private boolean flag = true;
//-1.声明ListState
private ListState<Long> offsetState = null; //用来存放offset
private Long offset = 0L;//用来存放offset的值
//-2.初始化/创建ListState
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
ListStateDescriptor<Long> stateDescriptor = new ListStateDescriptor<>("offsetState", Long.class);
offsetState = context.getOperatorStateStore().getListState(stateDescriptor);
}
//-3.使用state
@Override
public void run(SourceContext<String> ctx) throws Exception {
while (flag){
Iterator<Long> iterator = offsetState.get().iterator();
if(iterator.hasNext()){
offset = iterator.next();
}
offset += 1;
int subTaskId = getRuntimeContext().getIndexOfThisSubtask();
ctx.collect("subTaskId:"+ subTaskId + ",当前的offset值为:"+offset);
Thread.sleep(1000);
//模拟异常
if(offset % 5 == 0){
throw new Exception("bug出现了.....");
}
}
}
//-4.state持久化
//该方法会定时执行将state状态从内存存入Checkpoint磁盘目录中
@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
offsetState.clear();//清理内容数据并存入Checkpoint磁盘目录中
offsetState.add(offset);
}
@Override
public void cancel() {
flag = false;
}
}
}