Flink几个关键知识点

mapstate底层结构

package com.doit.flink.day06;/**
 * @Date 2022/2/20 15:52
 * @Created by JIA
 * @Description
 */
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.state.MapState;
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
/**
 * @ClassName MapStateDemo
 * @Date 2022/2/20 15:52
 * @Author jiajia
 * @Description TODO
 */
/**
 * 将ValueState的底层实现
 *
 * Flink的State分为两种：KeyedState(KeyBy之后对应的State)，和OperatorState（没有keyBy的State）
 *
 * ValueState是KeyedState中的一种
 *
 * 1.KeyedState底层是一个Map结构
 * 2.如果想要容错，必须开启checkpointing，并且按照Flink的状态编程API进行编程（将中间结果保存都Flink特殊的变量中）
 *
 * ValueState : Map<Key, Value>
 * MapState   : Map<Key, Map<k, v>>
 * ListState  : Map<Key, List<v>>
 *
 */
public class MapStateDemo {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //开启checkpoint
        env.enableCheckpointing(5000);

        //辽宁省,沈阳市,3000
        //辽宁省,大连市,4000
        //辽宁省,鞍山市,4000
        //河北省,廊坊市,2000
        //河北省,邢台市,3000
        //河北省,石家庄市,2000
        DataStreamSource<String> lines = env.socketTextStream("linux01", 8888, "\n", 5);

        //对数据进行整理
        SingleOutputStreamOperator<Tuple3<String, String, Integer>> tpStream = lines.map(new MapFunction<String, Tuple3<String, String, Integer>>() {
            @Override
            public Tuple3<String, String, Integer> map(String line) throws Exception {
                String[] fields = line.split(",");
                String province = fields[0];
                String city = fields[1];
                int money = Integer.parseInt(fields[2]);
                return Tuple3.of(province, city, money);
            }
        });

        //按照省份进行keyBy，将同一个省份的数据分到同一个分区中，并且按照城市累加金额
        KeyedStream<Tuple3<String, String, Integer>, String> keyedStream = tpStream.keyBy(t -> t.f0);
        SingleOutputStreamOperator<Tuple3<String, String, Integer>> res = keyedStream.map(new CityMoneyFunction());
        res.print();
        env.execute();

    }
    private static class CityMoneyFunction extends RichMapFunction<Tuple3<String,String,Integer>,Tuple3<String,String,Integer>>{
        private MapState<String,Integer>mapState;

        @Override
        public void open(Configuration parameters) throws Exception {
            //定义mapStateDescriptor
            MapStateDescriptor<String, Integer> stateDescriptor = new MapStateDescriptor<>("city-money-state", String.class, Integer.class);
            //初始化或恢复状态
            mapState = getRuntimeContext().getMapState(stateDescriptor);
        }

        @Override
        public Tuple3<String, String, Integer> map(Tuple3<String, String, Integer> input) throws Exception {
            String city = input.f1;
            Integer money = input.f2;
            Integer history = mapState.get(city);
            if (history==null){
                history=0;
            }
            money+=history;
            //更新状态
            mapState.put(city,money);
            //输出数据
            input.f2=money;
            return input;
        }
    }
}