flink应用二

transfrom

transform1

public class Transform1 {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStreamSource<String> inputStream = env.readTextFile("D:\\kb11\\flinkstu\\resource\\sensor.txt");
        SingleOutputStreamOperator<SensorReading> mapStream = inputStream.map(new MapFunction<String, SensorReading>() {
            @Override
            public SensorReading map(String s) throws Exception {
                String[] split = s.split(",");
                return new SensorReading(split[0], Long.parseLong(split[1]), Double.parseDouble(split[2]));
            }
        });
//        mapStream.print();
//        SingleOutputStreamOperator<Integer> mapStream2 = inputStream.map(line -> {
//            return line.length();
//        });
//        mapStream2.print();
//        SingleOutputStreamOperator<String> flatMapStream = inputStream.flatMap(new FlatMapFunction<String, String>() {
//            @Override
//            public void flatMap(String s, Collector<String> collector) throws Exception {
//                // sensor_1,111111111111111,36.2
//                String[] split = s.split(",");
//                for (String s1 : split) {
//                    collector.collect(s1);
//                }
//            }
//        });
//        flatMapStream.print();
//        SingleOutputStreamOperator<String> filterStream1 = inputStream.filter(new FilterFunction<String>() {
//            @Override
//            public boolean filter(String s) throws Exception {
//                return s.startsWith("sensor_7");
//            }
//        });
//        filterStream1.print();


        KeyedStream<SensorReading, Tuple> keyByStream = mapStream.keyBy(0);
        KeyedStream<SensorReading, Tuple> id = mapStream.keyBy("id");
        KeyedStream<SensorReading, String> keyedStream3 = mapStream.keyBy(sensorReading -> sensorReading.getId());
        KeyedStream<SensorReading, String> keyStream4 = mapStream.keyBy(SensorReading::getId);
        SingleOutputStreamOperator<SensorReading> resultStream = keyStream4.max("temperature");
        resultStream.print();
        env.execute("transform");
    }
}

transform2

public class Transform2 {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStreamSource<String> inputStream = env.readTextFile("D:\\kb11\\flinkstu\\resource\\sensor.txt");
        Properties prop = new Properties();
        prop.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.232.211:9092");
        prop.setProperty(ConsumerConfig.GROUP_ID_CONFIG,"sensor_group2");
        prop.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serizlization.StringDeserializer");
        prop.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serizlization.StringDeserializer");
        prop.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"latest");
        DataStreamSource<String> dataStream = env.addSource(new FlinkKafkaConsumer011<String>
                ("sensor", new SimpleStringSchema(), prop));
        SingleOutputStreamOperator<SensorReading> mapStream = inputStream.map(new MapFunction<String, SensorReading>() {
            @Override
            public SensorReading map(String s) throws Exception {
                String[] split = s.split(",");
                return new SensorReading(split[0], Long.parseLong(split[1]), Double.parseDouble(split[2]));
            }
        });
        //keyby返回keyedStream 可以进行聚合操作 min max minBy maxBy reduce ...
        KeyedStream<SensorReading, Tuple> keyedStream = mapStream.keyBy("id");
//        keyedStream.print();
        //返回值
//        SingleOutputStreamOperator<SensorReading> resultStream = keyedStream.max("temprature");
        //返回整个对象
//        SingleOutputStreamOperator<SensorReading> resultStream = keyedStream.maxBy("temprature");
//        resultStream.print();
        keyedStream.reduce(new ReduceFunction<SensorReading>() {
            @Override
            public SensorReading reduce(SensorReading sensorReading, SensorReading t1) throws Exception {
//                if (sensorReading.getTemprature()>t1.getTemprature()){
//                    return new SensorReading(sensorReading.getId(),sensorReading.getTimestamp(),
//                            t1.getTemprature());
//                }
                return new SensorReading(sensorReading.getId(),sensorReading.getTimestamp(),
                        Math.min(sensorReading.getTemperature(),t1.getTemperature()));
            }
        });
        SingleOutputStreamOperator<SensorReading> reduceStream = keyedStream.reduce((curSensorReading, newSensorReading) -> {
            return new SensorReading(curSensorReading.getId(), curSensorReading.getTimestamp(),
                    Math.min(curSensorReading.getTemperature(),newSensorReading.getTemperature()));
        });


        env.execute("transform2");
    }
}

transform3

public class Transform3 {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
//        DataStreamSource<String> inputStream = env.readTextFile("D:\\kb11\\flinkstu\\resource\\sensor.txt");
        Properties prop = new Properties();
        prop.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.232.211:9092");
        prop.setProperty(ConsumerConfig.GROUP_ID_CONFIG,"sensor_group2");
        prop.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serizlization.StringDeserializer");
        prop.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serizlization.StringDeserializer");
        prop.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"latest");
        DataStreamSource<String> dataStream = env.addSource(new FlinkKafkaConsumer011<String>
                ("sensor", new SimpleStringSchema(), prop));
        SingleOutputStreamOperator<SensorReading> mapStream = dataStream.map(new MapFunction<String, SensorReading>() {
            @Override
            public SensorReading map(String s) throws Exception {
                String[] split = s.split(",");
                return new SensorReading(split[0], Long.parseLong(split[1]), Double.parseDouble(split[2]));
            }
        });

        SplitStream<SensorReading> splitStream = mapStream.split(new OutputSelector<SensorReading>() {
            @Override
            public Iterable<String> select(SensorReading sensorReading) {
                if (sensorReading.getTemperature()> 38.0) {
                    return Collections.singletonList("high");
                } else if(sensorReading.getTemperature()<36.0) {
                    return Collections.singletonList("low");
                }else {
                    return Collections.singletonList("normal");
                }

            }
        });
        DataStream<SensorReading> high = splitStream.select("high");
        DataStream<SensorReading> normal = splitStream.select("normal");
        DataStream<SensorReading> low = splitStream.select("low");
        high.print("high");
        normal.print("normal");
        low.print("low");
        //union合流 数据结构类型一致
        DataStream<SensorReading> unionStream = high.union(low, normal);
        unionStream.print("union");
        //connect 合流 合并在一起的流 是不同的类型
        SingleOutputStreamOperator<Tuple2<String, Double>> warning = high.map(new MapFunction<SensorReading, Tuple2<String, Double>>() {
            @Override
            public Tuple2<String, Double> map(SensorReading sensorReading) throws Exception {
                return new Tuple2<>(sensorReading.getId(), sensorReading.getTemperature());
            }
        });
        ConnectedStreams<Tuple2<String, Double>, SensorReading> connect = warning.connect(normal);
        SingleOutputStreamOperator<Object> map = connect.map(new CoMapFunction<Tuple2<String, Double>, SensorReading, Object>() {
            @Override
            public Object map1(Tuple2<String, Double> stringDoubleTuple2) throws Exception {
                return new Tuple3<>(stringDoubleTuple2.f0, stringDoubleTuple2.f1, "发烧生病了");
            }

            @Override
            public Object map2(SensorReading sensorReading) throws Exception {
                return new Tuple2<>(sensorReading.getId(), "健康没有发烧");
            }
        });
        map.print("connect");
        env.execute("splitdemo");
    }
    private static class MyRichMapFunction extends RichMapFunction<SensorReading,Tuple2<String,Integer>>{
        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
        }

        @Override
        public void close() throws Exception {
            super.close();
        }

        @Override
        public Tuple2<String, Integer> map(SensorReading sensorReading) throws Exception {
            return null;
        }
    }

    private static class MyMapFunction implements MapFunction<SensorReading,Tuple2<String,Integer>>{
        @Override
        public Tuple2<String, Integer> map(SensorReading sensorReading) throws Exception {
            return null;
        }
    }
}