flink窗口函数ReduceFunction、AggregateFunction、ProcessFunction实例

1、ReduceFunction

增量,输入、状态、输出类型相同

import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;

import java.time.Duration;
import java.util.Random;

public class ReduceFunctionTest {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment executionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment();
executionEnvironment.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        executionEnvironment.getConfig().setAutoWatermarkInterval(100);

        DataStreamSource<Tuple2<String, Long>> streamSource = executionEnvironment.addSource(new SourceFunction<Tuple2<String, Long>>() {
            boolean flag = true;

            @Override
            public void run(SourceContext<Tuple2<String, Long>> sourceContext) throws Exception {
                String[] str = {"韩梅梅", "张三", "王五", "李四"};
                while (flag) {
                    Thread.sleep(1000);
                    int i = new Random().nextInt(4);
                    sourceContext.collect(new Tuple2<String, Long>(str[i], System.currentTimeMillis()));
                }
            }

            @Override
            public void cancel() {
                flag = false;
            }
        });

        streamSource.assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String,Long>>forBoundedOutOfOrderness(Duration.ofSeconds(1))
        .withTimestampAssigner(new SerializableTimestampAssigner<Tuple2<String, Long>>() {
            @Override
            public long extractTimestamp(Tuple2<String, Long> stringLongTuple2, long l) {
                return stringLongTuple2.f1;
            }
        })).map(new MapFunction<Tuple2<String, Long>, Tuple3<String, Long, Integer>>() {
            @Override
            public Tuple3<String, Long, Integer> map(Tuple2<String, Long> stringLongTuple2) throws Exception {
                System.out.println(stringLongTuple2.f0 + stringLongTuple2.f1);
                return new Tuple3<String, Long, Integer>(stringLongTuple2.f0,stringLongTuple2.f1,1);
            }
        }).keyBy(new KeySelector<Tuple3<String, Long, Integer>, String>() {
            @Override
            public String getKey(Tuple3<String, Long, Integer> stringIntegerTuple2) throws Exception {
                return stringIntegerTuple2.f0;
            }
        }).window(TumblingEventTimeWindows.of(Time.seconds(10)))
                .reduce(new ReduceFunction<Tuple3<String, Long, Integer>>() {
                    @Override
                    public Tuple3<String, Long, Integer> reduce(Tuple3<String, Long, Integer> stringIntegerTuple2, Tuple3<String, Long, Integer> t1) throws Exception {
                        return new Tuple3<String, Long, Integer>(stringIntegerTuple2.f0,stringIntegerTuple2.f1,stringIntegerTuple2.f2 + t1.f2);
                    }
                }).print();


        executionEnvironment.execute("reduce test");

    }
}

2、AggregateFunction

增量，输入、状态、输出类型可以不同

import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.AggregateFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;

import java.time.Duration;
import java.util.Random;

public class AggregateFunctionTest {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().setAutoWatermarkInterval(100);

        DataStreamSource<Tuple2<String, Long>> streamSource = env.addSource(new SourceFunction<Tuple2<String, Long>>() {
            boolean flag = true;

            @Override
            public void run(SourceContext<Tuple2<String, Long>> sourceContext) throws Exception {
                String[] str = {"韩梅梅", "张三", "王五", "李四"};
                while (flag) {
                    Thread.sleep(1000);
                    int i = new Random().nextInt(4);
                    sourceContext.collect(new Tuple2<String, Long>(str[i], System.currentTimeMillis()));
                }
            }

            @Override
            public void cancel() {
                flag = false;
            }
        });

        streamSource.assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String,Long>>forBoundedOutOfOrderness(Duration.ofSeconds(1))
        .withTimestampAssigner(new SerializableTimestampAssigner<Tuple2<String, Long>>() {
            @Override
            public long extractTimestamp(Tuple2<String, Long> stringLongTuple2, long l) {
                return stringLongTuple2.f1;
            }
        })).map(new MapFunction<Tuple2<String, Long>, Tuple2<String, Integer>>() {
            @Override
            public Tuple2<String, Integer> map(Tuple2<String, Long> stringLongTuple2) throws Exception {
                return new Tuple2<String,Integer>(stringLongTuple2.f0,1);
            }
        }).keyBy(new KeySelector<Tuple2<String, Integer>, String>() {
            @Override
            public String getKey(Tuple2<String, Integer> stringIntegerTuple2) throws Exception {
                return stringIntegerTuple2.f0;
            }
        }).window(TumblingEventTimeWindows.of(Time.seconds(10)))
                .aggregate(new AggregateFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>>() {

                    //存储中间状态state，窗口初始化时调用
                    @Override
                    public Tuple2<String, Integer> createAccumulator() {
                        return new Tuple2<String, Integer>("",0);
                    }

                    //窗口来新元素时调用
                    @Override
                    public Tuple2<String, Integer> add(Tuple2<String, Integer> stringIntegerTuple2, Tuple2<String, Integer> stringIntegerTuple22) {
                        return new Tuple2<String, Integer>(stringIntegerTuple2.f0,stringIntegerTuple2.f1 + stringIntegerTuple22.f1);
                    }

                    //获取最后结果
                    @Override
                    public Tuple2<String, Integer> getResult(Tuple2<String, Integer> stringIntegerTuple2) {
                        return stringIntegerTuple2;
                    }

                    //合并两个state，窗口类型为session的时候使用，两个session窗口有可能合并为一个
                    @Override
                    public Tuple2<String, Integer> merge(Tuple2<String, Integer> stringIntegerTuple2, Tuple2<String, Integer> acc1) {
                        return null;
                    }
                }).print();

        env.execute("aggregate test");


    }
}

3、ProcessFunction

窗口数据全量计算，输入、输出类型可以不同

import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;

import java.time.Duration;
import java.util.Random;

public class ProcessFunctionTest {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().setAutoWatermarkInterval(100);

        DataStreamSource<Tuple2<String, Long>> streamSource = env.addSource(new SourceFunction<Tuple2<String, Long>>() {
            boolean flag = true;

            @Override
            public void run(SourceContext<Tuple2<String, Long>> sourceContext) throws Exception {
                String[] str = {"韩梅梅", "张三", "王五", "李四"};
                while (flag) {
                    Thread.sleep(1000);
                    int i = new Random().nextInt(4);
                    sourceContext.collect(new Tuple2<String, Long>(str[i], System.currentTimeMillis()));
                }
            }

            @Override
            public void cancel() {
                flag = false;
            }
        });

        streamSource.assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String,Long>>forBoundedOutOfOrderness(Duration.ofSeconds(1))
                .withTimestampAssigner(new SerializableTimestampAssigner<Tuple2<String, Long>>() {
                    @Override
                    public long extractTimestamp(Tuple2<String, Long> stringLongTuple2, long l) {
                        return stringLongTuple2.f1;
                    }
        })).map(new MapFunction<Tuple2<String, Long>, Tuple2<String, Integer>>() {
            @Override
            public Tuple2<String, Integer> map(Tuple2<String, Long> stringLongTuple2) throws Exception {
                return new Tuple2<String,Integer>(stringLongTuple2.f0,1);
            }
        }).keyBy(new KeySelector<Tuple2<String, Integer>, String>() {
            @Override
            public String getKey(Tuple2<String, Integer> stringIntegerTuple2) throws Exception {
                return stringIntegerTuple2.f0;
            }
        }).window(TumblingEventTimeWindows.of(Time.seconds(10)))
//输入，输出，key，窗口类型
          .process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {

//key，上下文，窗口中的所有元素，返回收集器
                    @Override
                    public void process(String key, Context context, Iterable<Tuple2<String, Integer>> elements, Collector<Tuple2<String, Integer>> out) throws Exception {
                        int count = 0;
                        for (Tuple2<String, Integer> value : elements ) {
                            count = count + value.f1;
                        }
                        out.collect(new Tuple2<String, Integer>(key,count));
                    }
                }).print();

        env.execute("process test");
    }
}