加微信拉微信交流群
微信:weixin605405145
KeyedState
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
public class TestKeyedState {
public static void main(String[] args) throws Exception{
StreamExecutionEnvironment env=StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Tuple2<Long,Long>> inputStream=env.fromElements(
Tuple2.of(1L,4L),
Tuple2.of(2L,3L),
Tuple2.of(3L,1L),
Tuple2.of(1L,2L),
Tuple2.of(3L,2L),
Tuple2.of(1L,2L),
Tuple2.of(2L,2L),
Tuple2.of(2L,9L)
);
inputStream
.keyBy(0)
.flatMap(new CountWithKeyedState())
.setParallelism(10)
.print();
env.execute();
}
}
CountWithKeyedState
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.state.StateTtlConfig;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.util.Collector;
public class CountWithKeyedState extends RichFlatMapFunction<Tuple2<Long,Long>,Tuple2<Long,Long>> {
private transient ValueState<Tuple2<Long,Long>> sum;
@Override
public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
Tuple2<Long,Long> currentSum=sum.value();
if(null==currentSum){
currentSum=Tuple2.of(0L,0L);
}
currentSum.f0+=1;
currentSum.f1+=value.f1;
sum.update(currentSum);
if(currentSum.f0>=3){
out.collect(Tuple2.of(value.f0,currentSum.f1/currentSum.f0));
sum.clear();
}
}
@Override
public void open(Configuration parameters) throws Exception {
// StateTtlConfig ttlConfig = StateTtlConfig
// .newBuilder(Time.seconds(1))
// .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
// .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
// .build();
/**
* 注意这里仅仅用了状态,但是没有利用状态来容错
*/
ValueStateDescriptor<Tuple2<Long,Long>> descriptor=
new ValueStateDescriptor<>(
"avgState",
TypeInformation.of(new TypeHint<Tuple2<Long, Long>>() {})
);
// descriptor.enableTimeToLive(ttlConfig);
sum=getRuntimeContext().getState(descriptor);
}
}
OperatorState
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
import org.apache.flink.runtime.state.StateBackend;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import java.util.concurrent.TimeUnit;
public class TestOperatorState {
public static void main(String[] args) throws Exception{
StreamExecutionEnvironment env=StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
//设置checkpoint
env.enableCheckpointing(60000L);
CheckpointConfig checkpointConfig=env.getCheckpointConfig();
checkpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
checkpointConfig.setMinPauseBetweenCheckpoints(30000L);
checkpointConfig.setCheckpointTimeout(10000L);
checkpointConfig.setFailOnCheckpointingErrors(true);
checkpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION);
// StateBackend backend=new FsStateBackend(
// "hdfs://namenode:40010/flink/checkpoints",
// false);
// StateBackend backend=new MemoryStateBackend(10*1024*1024,false);
StateBackend backend=new RocksDBStateBackend(
"hdfs://namenode:40010/flink/checkpoints",
true);
env.setStateBackend(backend);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(
3, // number of restart attempts
Time.of(10, TimeUnit.SECONDS) // delay
));
DataStream<Long> inputStream=env.fromElements(1L,2L,3L,4L,5L,1L,3L,4L,5L,6L,7L,1L,4L,5L,3L,9L,9L,2L,1L);
inputStream.flatMap(new CountWithOperatorState())
.setParallelism(1)
.print();
env.execute();
}
}
CountWithOperatorState
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.FunctionInitializationContext;
import org.apache.flink.runtime.state.FunctionSnapshotContext;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
import org.apache.flink.util.Collector;
import java.util.ArrayList;
import java.util.List;
/**
*
* 想知道两次事件1之间,一共发生多少次其他事件,分别是什么事件
*
* 事件流:1 2 3 4 5 1 3 4 5 6 7 1 4 5 3 9 9 2 1...
* 输出:
* (4,2 3 4 5)
* (5,3 4 5 6 7)
* (6,4 5 6 9 9 2)
*/
public class CountWithOperatorState extends RichFlatMapFunction<Long,Tuple2<Integer,String>> implements CheckpointedFunction {
private transient ListState<Long> checkPointCountList;
/**
* 原始状态
*/
private List<Long> listBufferElements;
@Override
public void flatMap(Long value, Collector<Tuple2<Integer,String>> out) throws Exception {
if(value == 1){
if(listBufferElements.size()>0){
StringBuffer buffer=new StringBuffer();
for(Long item:listBufferElements){
buffer.append(item+" ");
}
out.collect(Tuple2.of(listBufferElements.size(),buffer.toString()));
listBufferElements.clear();
}
}else{
listBufferElements.add(value);
}
}
@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
checkPointCountList.clear();
for(Long item:listBufferElements){
checkPointCountList.add(item);
}
}
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
ListStateDescriptor<Long> listStateDescriptor=
new ListStateDescriptor<Long>("checkPointCountList",TypeInformation.of(new TypeHint<Long>() {}));
checkPointCountList=context.getOperatorStateStore().getListState(listStateDescriptor);
if(context.isRestored()){
for(Long element:checkPointCountList.get()){
listBufferElements.add(element);
}
}
}
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
listBufferElements=new ArrayList<>();
}
}