《初级》Flink 窗口函数练习

加微信拉微信交流群

微信：weixin605405145

ProcessWinFunOnWindow

import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
import org.apache.flink.util.Collector;


public class TestProcessWinFunOnWindow {
    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env=StreamExecutionEnvironment.getExecutionEnvironment();

        DataStream<Tuple3<String,String,Long>> input=env.fromElements(ENGLISH_TRANSCRIPT);

        //求各班级英语平均分
        DataStream<Double> avgScore=input.keyBy(0).countWindow(2).process(new MyProcessWindowFunction());
        avgScore.print();
        env.execute();
    }

    /**
     * 英语成绩
     */
    public static final Tuple3[] ENGLISH_TRANSCRIPT = new Tuple3[] {
            Tuple3.of("class1","张三",100L),
            Tuple3.of("class1","李四",78L),
            Tuple3.of("class1","王五",99L),
            Tuple3.of("class2","赵六",81L),
            Tuple3.of("class2","钱七",59L),
            Tuple3.of("class2","马二",97L)
    };

    private static class MyProcessWindowFunction
            extends ProcessWindowFunction<Tuple3<String,String,Long>, Double, Tuple, GlobalWindow> {
        @Override
        public void process(Tuple tuple, Context context, Iterable<Tuple3<String, String, Long>> elements, Collector<Double> out) throws Exception {
            long sum=0;
            long count = 0;
            for (Tuple3<String, String, Long> in: elements) {
                sum+=in.f2;
                count++;
            }
            //out.collect("Window: " + context.window() + "count: " + count);
            out.collect((double) (sum/count));
        }
    }
}

 

AggFunctionOnWindow

import org.apache.flink.api.common.functions.AggregateFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;


public class TestAggFunctionOnWindow {
    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env=StreamExecutionEnvironment.getExecutionEnvironment();

        DataStream<Tuple3<String,String,Long>> input=env.fromElements(ENGLISH_TRANSCRIPT);

        //求各班级英语平均分
        DataStream<Double> avgScore=input.
                keyBy(0).
                countWindow(2).
                aggregate(new AverageAggregate());
        avgScore.print();

        env.execute();
    }

    /**
     * 英语成绩
     */
    public static final Tuple3[] ENGLISH_TRANSCRIPT = new Tuple3[] {
            Tuple3.of("class1","张三",100L),
            Tuple3.of("class1","李四",78L),
            Tuple3.of("class1","王五",99L),
            Tuple3.of("class2","赵六",81L),
            Tuple3.of("class2","钱七",59L),
            Tuple3.of("class2","马二",97L)
    };

    /**
     * 求第二个字段的平均值(sum/count)
     */
    private static class AverageAggregate
            implements AggregateFunction<Tuple3<String, String,Long>, Tuple2<Long, Long>, Double> {
        /**
         * 创建累加器来保存中间状态(sum和count)
         * @return
         */
        @Override
        public Tuple2<Long, Long> createAccumulator() {
            return new Tuple2<>(0L, 0L);
        }

        /**
         * 将元素添加到累加器并返回新的累加器value
         * @param value
         * @param accumulator
         * @return
         */
        @Override
        public Tuple2<Long, Long> add(Tuple3<String, String, Long> value, Tuple2<Long, Long> accumulator) {
            //来一个元素计算一下sum和count并保存中间结果到累加器
            return new Tuple2<>(accumulator.f0 + value.f2, accumulator.f1 + 1L);
        }

        /**
         * 从累加器提取结果
         * @param accumulator
         * @return
         */
        @Override
        public Double getResult(Tuple2<Long, Long> accumulator) {
            return ((double) accumulator.f0) / accumulator.f1;
        }

        @Override
        public Tuple2<Long, Long> merge(Tuple2<Long, Long> a, Tuple2<Long, Long> b) {
            return new Tuple2<>(a.f0 + b.f0, a.f1 + b.f1);
        }

    }
}

 

 

ReduceFunctionOnWindowAll

import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;


public class TestReduceFunctionOnWindowAll {
    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env=StreamExecutionEnvironment.getExecutionEnvironment();

        DataStream<Tuple3<String,String,Integer>> input=env.fromElements(ENGLISH_TRANSCRIPT);

        //求英语总分
        DataStream<Tuple3<String,String,Integer>> totalPoints=input.keyBy(0).countWindow(2).reduce(new ReduceFunction<Tuple3<String, String, Integer>>() {
            @Override
            public Tuple3<String, String, Integer> reduce(Tuple3<String, String, Integer> value1, Tuple3<String, String, Integer> value2) throws Exception {
                return new Tuple3<>(value1.f0, value1.f1 ,value1.f2+value2.f2);
            }
        });
        totalPoints.print();

        env.execute();
    }

    /**
     * 语文成绩
     */
    public static final Tuple3[] ENGLISH_TRANSCRIPT = new Tuple3[] {
            Tuple3.of("class1","张三",100),
            Tuple3.of("class1","李四",78),
            Tuple3.of("class1","王五",99),
            Tuple3.of("class2","赵六",81),
            Tuple3.of("class2","钱七",59),
            Tuple3.of("class2","马二",97)
    };
}

 

ReduceFunctionOnWindow

import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class TestReduceFunctionOnWindow {
    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env=StreamExecutionEnvironment.getExecutionEnvironment();

        DataStream<Tuple3<String,String,Integer>> input=env.fromElements(ENGLISH_TRANSCRIPT);

        //求各班级英语总分
        DataStream<Tuple3<String,String,Integer>> totalPoints=input.keyBy(0).countWindow(2).reduce(new ReduceFunction<Tuple3<String, String, Integer>>() {
            @Override
            public Tuple3<String, String, Integer> reduce(Tuple3<String, String, Integer> value1, Tuple3<String, String, Integer> value2) throws Exception {
                return new Tuple3<>(value1.f0, value1.f1 ,value1.f2+value2.f2);
            }
        });
        totalPoints.print();

        env.execute();
    }

    /**
     * 英语成绩
     */
    public static final Tuple3[] ENGLISH_TRANSCRIPT = new Tuple3[] {
            Tuple3.of("class1","张三",100),
            Tuple3.of("class1","李四",78),
            Tuple3.of("class1","王五",99),
            Tuple3.of("class2","赵六",81),
            Tuple3.of("class2","钱七",59),
            Tuple3.of("class2","马二",97)
    };
}