Flink Connect、Split、SideOutput 实例

1、Connect

合并两个数据流，不是关联。

import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

import java.util.Random;

public class ConnectTest {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.getConfig().setAutoWatermarkInterval(100l);

        DataStreamSource<Tuple3<String, Integer, Long>> tuple3DataStreamSource1 = env.addSource(new SourceFunction<Tuple3<String, Integer, Long>>() {
            boolean flag = true;

            @Override
            public void run(SourceContext<Tuple3<String, Integer, Long>> ctx) throws Exception {
                String[] str = {"水阀1", "水阀2", "水阀3"};
                while (flag) {
                    int i = new Random().nextInt(3);
                    // 温度
                    int temperature = new Random().nextInt(100);
                    Thread.sleep(1000l);
                    // 设备号、温度、事件时间
                    ctx.collect(new Tuple3<String, Integer, Long>(str[i], temperature, System.currentTimeMillis()));
                }
            }

            @Override
            public void cancel() {
                flag = false;
            }
        });

        DataStreamSource<Tuple3<String, Integer, Long>> tuple3DataStreamSource2 = env.addSource(new SourceFunction<Tuple3<String, Integer, Long>>() {
            boolean flag = true;

            @Override
            public void run(SourceContext<Tuple3<String, Integer, Long>> ctx) throws Exception {
                String[] str = {"水阀4", "水阀5", "水阀6"};
                while (flag) {
                    int i = new Random().nextInt(3);
                    // 温度
                    int temperature = new Random().nextInt(100);
                    Thread.sleep(1000l);
                    // 设备号、温度、事件时间
                    ctx.collect(new Tuple3<String, Integer, Long>(str[i], temperature, System.currentTimeMillis()));
                }
            }

            @Override
            public void cancel() {
                flag = false;
            }
        });

        SingleOutputStreamOperator<Tuple3<String, Integer, Long>> filter1 = tuple3DataStreamSource1.filter(new FilterFunction<Tuple3<String, Integer, Long>>() {
            @Override
            public boolean filter(Tuple3<String, Integer, Long> stringIntegerLongTuple3) throws Exception {
                return stringIntegerLongTuple3.f0.equals("水阀1");
            }
        });

        SingleOutputStreamOperator<Tuple3<String, Integer, Long>> filter2 = tuple3DataStreamSource2.filter(new FilterFunction<Tuple3<String, Integer, Long>>() {
            @Override
            public boolean filter(Tuple3<String, Integer, Long> stringIntegerLongTuple3) throws Exception {
                return stringIntegerLongTuple3.f0.equals("水阀4");
            }
        });

        filter1.connect(filter2).map(new CoMapFunction<Tuple3<String,Integer,Long>,Tuple3<String,Integer,Long>,Tuple2<String,Integer>>(){

            @Override
            public Tuple2<String, Integer> map1(Tuple3<String, Integer, Long> value) throws Exception {
                return new Tuple2<String, Integer>(value.f0,value.f1);
            }

            @Override
            public Tuple2<String, Integer> map2(Tuple3<String, Integer, Long> value) throws Exception {
                return new Tuple2<String, Integer>(value.f0,value.f1);
            }

        }).print();

        env.execute("connect");

    }
}

2、Split和SideOutput 

拆分数据流，并侧输出

import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
import java.util.Random;

public class SplitTest {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().setAutoWatermarkInterval(100l);

        DataStreamSource<Tuple3<String, Integer, Long>> tuple3DataStreamSource1 = env.addSource(new SourceFunction<Tuple3<String, Integer, Long>>() {
            boolean flag = true;

            @Override
            public void run(SourceContext<Tuple3<String, Integer, Long>> ctx) throws Exception {
                String[] str = {"水阀1", "水阀2", "水阀3"};
                while (flag) {
                    int i = new Random().nextInt(3);
                    // 温度
                    int temperature = new Random().nextInt(100);
                    Thread.sleep(1000l);
                    // 设备号、温度、事件时间
                    ctx.collect(new Tuple3<String, Integer, Long>(str[i], temperature, System.currentTimeMillis()));
                }
            }

            @Override
            public void cancel() {
                flag = false;
            }
        });

        // 侧输出流标签
        OutputTag<Tuple3<String, Integer, Long>> sf1 = new OutputTag<Tuple3<String, Integer, Long>>("sf1", TypeInformation.of(new TypeHint<Tuple3<String, Integer, Long>>() {
        }));

        OutputTag<Tuple3<String, Integer, Long>> sf2 = new OutputTag<Tuple3<String, Integer, Long>>("sf2", TypeInformation.of(new TypeHint<Tuple3<String, Integer, Long>>() {
        }));

        SingleOutputStreamOperator<Tuple3<String, Integer, Long>> process = tuple3DataStreamSource1.process(new ProcessFunction<Tuple3<String, Integer, Long>, Tuple3<String, Integer, Long>>() {

            @Override
            public void processElement(Tuple3<String, Integer, Long> value, Context ctx, Collector<Tuple3<String, Integer, Long>> out) throws Exception {
                if (value.f0.equals("水阀1")) {
                    ctx.output(sf1, value);
                } else if (value.f0.equals("水阀2")) {
                    ctx.output(sf2, value);
                }
            }
        });

        // 获取侧输出流
        process.getSideOutput(sf1).print("水阀1");

        process.getSideOutput(sf2).print("水阀2");

        // 已过时，并且split不能二次拆分
//        SplitStream<Tuple3<String, Integer, Long>> split = tuple3DataStreamSource1.split(new OutputSelector<Tuple3<String, Integer, Long>>() {
//            @Override
//            public Iterable<String> select(Tuple3<String, Integer, Long> value) {
//                if (value.f0.equals("水阀1")) {
//                    return Collections.singletonList("水阀1"); // 该流的标识
//                } else {
//                    return Collections.singletonList("水阀"); // 该流的标识
//                }
//
//            }
//        });
//
//        split.select("水阀1").print();

        env.execute("split");

    }
}