java Flink（十三）Flink底层Process使用Demo以及示例 ---访问时间戳、watermark 以及注册定时事件以及侧输出流

Process:

我们之前学习的转换算子是无法访问事件的时间戳信息和水位线信息的。而这 在一些应用场景下，极为重要。例如 MapFunction 这样的 map 转换算子就无法访问 时间戳或者当前事件的事件时间。 基于此，DataStream API 提供了一系列的 Low-Level 转换算子。可以访问时间 戳、watermark 以及注册定时事件。还可以输出特定的一些事件，例如超时事件等。 Process Function 用来构建事件驱动的应用以及实现自定义的业务逻辑(使用之前的 window 函数和转换算子无法实现)。

Process最常用的功能就是访问时间 戳、watermark 以及注册定时事件以及侧输出流

Demo

public class WindowTest1 {
    public static void main(String[] args) throws Exception {
        //获取当前执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //设置并行度
        env.setParallelism(1);
        //设置时间语义为时间时间
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //从文件读取数据 ***demo模拟的文本流处理，其实不该用文本流处理，因为读取文本根本不许要分桶，速度过快了
        //DataStream<String> inputStream = env.readTextFile("D:\\idle\\FlinkTest\\src\\main\\resources");
        //从kafka读取数据
        DataStream<String> inputStream = env.addSource(new FlinkKafkaConsumer011<String>(kafkaConsumerTopic,
                new SimpleStringSchema(), KafkaConsumerPro.getKafkaConsumerPro()));

        //转换成SensorReading类型
        DataStream<SensorReading> dataStream = inputStream.map(new MapFunction<String, SensorReading>() {
            public SensorReading map(String line) throws Exception {
                String[] fields = line.split(",");
                return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
            }
        });

        //定义一个有状态的map操作. 基于key，先做keyBy
       dataStream.keyBy("id")
                .process(new MyProcess())
                .print();
    }

    public static class MyProcess extends KeyedProcessFunction<Tuple, SensorReading, Integer>{
        ValueState<Long> tsTimerState;

        @Override
        public void open(Configuration parameters) throws Exception {
            tsTimerState = getRuntimeContext().getState(new ValueStateDescriptor<Long>("ts-timer", Long.class));
        }

        public void processElement(SensorReading value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value.getId().length()); //输出数据
            //使用上下文ctx
            ctx.timestamp(); //获取当前时间戳
            ctx.getCurrentKey(); //获取当前key
            //ctx.output();//测输出流
            ctx.timerService().registerProcessingTimeTimer((value.getTimestamp()+10)*1000);//时间服务 当前时间戳往后10秒
            tsTimerState.update((value.getTimestamp()+10)*1000);//保存时间，方便删除时使用
            ctx.timerService().registerProcessingTimeTimer(tsTimerState.value());
        }

        @Override
        public void onTimer(long timestamp, OnTimerContext ctx, Collector<Integer> out) throws Exception {
            System.out.println(timestamp+" 定时器触发");

        }
        @Override
        public void close() throws Exception {
            tsTimerState.clear();
        }
    }

}

使用示例 

题目：温度10s内连续上升则报警

public class WindowTest1 {
    public static void main(String[] args) throws Exception {
        //获取当前执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //设置并行度
        env.setParallelism(1);
        //设置时间语义为时间时间
        //env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //从文件读取数据 ***demo模拟的文本流处理，其实不该用文本流处理，因为读取文本根本不许要分桶，速度过快了
        //DataStream<String> inputStream = env.readTextFile("D:\\idle\\FlinkTest\\src\\main\\resources");
        //从kafka读取数据
        DataStream<String> inputStream = env.addSource(new FlinkKafkaConsumer011<String>(kafkaConsumerTopic,
                new SimpleStringSchema(), KafkaConsumerPro.getKafkaConsumerPro()));

        //转换成SensorReading类型
        DataStream<SensorReading> dataStream = inputStream.map(new MapFunction<String, SensorReading>() {
            public SensorReading map(String line) throws Exception {
                String[] fields = line.split(",");
                return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
            }
        });

        //定义一个有状态的map操作. 基于key，先做keyBy
       dataStream.keyBy("id")
                .process(new MyProcess(10))
                .print();
    }

    public static class MyProcess extends KeyedProcessFunction<Tuple, SensorReading, String>{
        //定义接收当前统计的时间间隔
        private Integer ConCount;
        public MyProcess(Integer ConCount) {
            this.ConCount = ConCount;
        }

        //定义状态 保存上次温度值、定时器时间戳
        private ValueState<Double> lastTempStates;
        private ValueState<Long> tsTimerState;
        @Override
        public void open(Configuration parameters) throws Exception {
           //注册状态
            lastTempStates = getRuntimeContext().getState(new ValueStateDescriptor<Double>("last-temp", Double.class, Double.MIN_VALUE));
            tsTimerState = getRuntimeContext().getState(new ValueStateDescriptor<Long>("ts-timer", Long.class));
        }

        public void processElement(SensorReading value, Context ctx, Collector<String> out) throws Exception {
            //取出状态
            Double lastTemp = lastTempStates.value();
            Long timerTs = tsTimerState.value();
            //如果温度上升并且没有定时器，注册10s后定时器，开始等待
            if(value.getTimestamp()>lastTemp && timerTs==null){
                //计算定时器时间戳
                Long ts = ctx.timerService().currentProcessingTime() + ConCount * 1000L;
                //注册定时器
                ctx.timerService().registerProcessingTimeTimer(ts);
                tsTimerState.update(ts);
            }
            //如果温度下降，删除定时器
            else if(value.getTimestamp()<lastTemp && timerTs!=null){
                ctx.timerService().deleteProcessingTimeTimer(timerTs);
                tsTimerState.clear();
            }
            //更新温度状态
            lastTempStates.update(value.getTemperature());

        }

        @Override
        public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
            System.out.println(timestamp+" 定时器触发");
            //定时器触发，输出报警信息
            out.collect("传感器"+ctx.getCurrentKey().getField(0)+"温度值连续"+ConCount+"上升");
            tsTimerState.clear();

        }
        @Override
        public void close() throws Exception {
            lastTempStates.clear();
            tsTimerState.clear();
        }
    }

}

测输出流：

题目：打印温度低于30度数据

public class WindowTest1 {
    public static void main(String[] args) throws Exception {
        //获取当前执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //设置并行度
        env.setParallelism(1);
        DataStream<String> inputStream = env.addSource(new FlinkKafkaConsumer011<String>(kafkaConsumerTopic,
                new SimpleStringSchema(), KafkaConsumerPro.getKafkaConsumerPro()));

        //转换成SensorReading类型
        DataStream<SensorReading> dataStream = inputStream.map(new MapFunction<String, SensorReading>() {
            public SensorReading map(String line) throws Exception {
                String[] fields = line.split(",");
                return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
            }
        });
        //定义一个OutputTag,用来表示测输出流低温度流
        final OutputTag<SensorReading> lowTempTag = new OutputTag<SensorReading>("low-temp"){};
        //输出高温度流
        SingleOutputStreamOperator<SensorReading> highTempStream = dataStream.process(new ProcessFunction<SensorReading, SensorReading>() {
            public void processElement(SensorReading value, Context ctx, Collector<SensorReading> out) throws Exception {
                //如果温度大于30 输出到高温度流
                if(value.getTimestamp() > 30){
                    out.collect(value);
                }
                else {//否则输出到低温度流
                    ctx.output(lowTempTag, value);
                }
            }
        });
        highTempStream.print("high-temp");
        highTempStream.getSideOutput(lowTempTag).print("low-temp");

    }
}