Flink-1.11.2-code_demo

最新推荐文章于 2023-12-20 19:01:12 发布
最新推荐文章于 2023-12-20 19:01:12 发布
阅读量235 收藏 1
点赞数
分类专栏： Flink1.11.2 文章标签： flink
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_31316733/article/details/113838225
版权
本文详细介绍了Apache Flink 1.11.2版本中的数据源（FromCollection, FromFile, FromKafka, CustomizeSource）、转换操作（T_Map_FlatMap, T_Filter等）、窗口处理（time_window, count_window）以及水印（AccendWatermark, SequenceWatermark）和输出（ToKafka, ToES, ToRedis, CustomizeSink）。通过实例演示了Flink的核心功能。" 115603024,6260336,一文学懂经典算法：详解折半查找,"['经典算法', '查找算法', '数据结构', '算法效率', 'Java实现']
摘要由CSDN通过智能技术生成

Flink-1.11.2-code_demo

一.source

1.1.FromCollection

package com.flink.source;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import java.util.ArrayList;
public class FromCollection {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        ArrayList<WaterSensor> waterSensorsLists = new ArrayList<>();
        waterSensorsLists.add(new WaterSensor("001", 123456L, 100));
        waterSensorsLists.add(new WaterSensor("002", 123457L, 200));
        waterSensorsLists.add(new WaterSensor("003", 123458L, 300));
        waterSensorsLists.add(new WaterSensor("004", 123459L, 400));
        waterSensorsLists.add(new WaterSensor("005", 123460L, 500));
        DataStreamSource<WaterSensor> ds = env.fromCollection(waterSensorsLists);
        ds.print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

1.2.FromFile

package com.flink.source;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
public class FromFile {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //From local
        //DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        //From hdfs
        DataStreamSource<String> ds_hdfs = env.readTextFile("hdfs://hadoop162:8020/data/water_sensor.txt");
        ds_hdfs.print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

1.3.FromKafka

package com.flink.source;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import java.util.Properties;
public class FromKafka {
    public static void main(String[] args) {
        Properties props = new Properties();
        props.put("bootstrap.servers", "192.168.1.162:9092");//kafka集群，broker-list
        props.put("acks", "all");
        props.put("retries", 1);//批次大小
        props.put("linger.ms", 1);//等待时重试次数
        props.put("batch.size", 16384);//间
        props.put("buffer.memory", 33554432);//RecordAccumulator缓冲区大小
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("auto.offset.reset", "earliest");  //earliest,latest
        Producer<String, String> producer = new KafkaProducer<>(props);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> dss = env.addSource(new FlinkKafkaConsumer011<String>(
                "water_sensor"
                ,new SimpleStringSchema()
                ,props
        ));
        dss.print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

1.4.1.CustomizeSource

package com.flink.source;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import java.util.Random;
public class CustomizeSource implements SourceFunction<WaterSensor> {
    boolean flag = true;
    @Override
    public void run(SourceContext<WaterSensor> ctx) throws Exception {
        Random random = new Random();
        while (flag) {
            ctx.collect(
                    new WaterSensor(
                            "sensor_" + (random.nextInt(3) + 1),
                            System.currentTimeMillis(),
                            random.nextInt(10) + 40
                    )
            );
            Thread.sleep(1000L);
        }
    }
    @Override
    public void cancel() {
        flag=false;
    }
}

1.4.1.FromCustomizeSource

package com.flink.source;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
public class FromCustomizeSource {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<WaterSensor> ds = env.addSource(new CustomizeSource());
        ds.print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

二.transform

2.1.T_Map_FlatMap

package com.flink.transform;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.util.Collector;
public class T_Map_FlatMap {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\word_count.txt");
        //        ds.flatMap((s, out) -> {
//            String[] line = s.split(" ");
//            for (String s1 : line) {
//                out.collect(s1);
//            }
//        })
//        .print();
        ds.map((in)->{
            String[] line = in.split(" ");
            String out="";
            for (String s : line) {
                out+=s+"\n";
            }
            return out;
        })
                .print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

2.2.T_Filter

package com.flink.transform;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
public class T_Filter {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\word_count.txt");

            ds.flatMap(new FlatMapFunction<String, String>() {
                @Override
                public void flatMap(String in, Collector<String> out) throws Exception {
                    String[] line = in.split(" ");
                    for (String word : line) {
                        out.collect(word);
                    }
                }
            })
                .filter((in)->in.toString().contains("a"))
                .print();

        try {
            env.execute("filter");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

2.3.T_KeyBy

package com.flink.transform;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
public class T_KeyBy {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\word_count.txt");
        ds.flatMap(new FlatMapFunction<String, Tuple2<String,Integer>>() {
            @Override
            public void flatMap(String in, Collector<Tuple2<String,Integer>> out) throws Exception {
                String[] line = in.split(" ");
                for (String s : line) {
                    out.collect(new Tuple2(s,1));
                }
            }
        })
                .keyBy(0)
                .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
                    @Override
                    public Tuple2<String, Integer> reduce(Tuple2<String, Integer> now, Tuple2<String, Integer> next) throws Exception {
                        return new Tuple2<String,Integer>(now.f0,new Integer(now.f1+next.f1));
                    }
                })
                .print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

2.4.T_Shuffle

package com.flink.transform;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
public class T_Shuffle {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\word_count.txt");
        ds.flatMap(new FlatMapFunction<String, Tuple2<String,Integer>>() {
            @Override
            public void flatMap(String in, Collector<Tuple2<String,Integer>> out) throws Exception {
                String[] line = in.split(" ");
                for (String s : line) {
                    out.collect(new Tuple2(s,1));
                }
            }
        })
                .shuffle()
                .print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

2.5.T_Split_Select

package com.flink.transform;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
import java.util.Arrays;
public class T_Split_Select {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\word_count.txt");
        ds.flatMap(new FlatMapFunction<String, Tuple2<String,Integer>>() {
            @Override
            public void flatMap(String in, Collector<Tuple2<String,Integer>> out) throws Exception {
                String[] line = in.split(" ");
                for (String s : line) {
                    out.collect(new Tuple2(s,1));
                }
            }
        })
                .keyBy(0)
                .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
                    @Override
                    public Tuple2<String, Integer> reduce(Tuple2<String, Integer> now, Tuple2<String, Integer> next) throws Exception {
                        return new Tuple2<String,Integer>(now.f0,new Integer(now.f1+next.f1));
                    }
                })
                .split(new OutputSelector<Tuple2<String, Integer>>() {
                    @Override
                    public Iterable<String> select(Tuple2<String, Integer> in) {
                        if(in.f0.contains("a")){
                            return Arrays.asList("Contains_a");
                        }else  if(in.f0.contains("b")){
                            return Arrays.asList("Contains_b");
                        }else  if(in.f0.contains("c")){
                            return Arrays.asList("Contains_c");
                        }else {
                            return  Arrays.asList("other");
                        }
                    }
                })
                .select("Contains_b")
                .print("Contains_b");

        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

2.6.T_Connect_Union

package com.flink.transform;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.ConnectedStreams;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SplitStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
import java.util.Arrays;
public class T_Connect_Union {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\word_count.txt");
        SplitStream<Tuple2<String, Integer>> streamSplits = ds.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String in, Collector<Tuple2<String, Integer>> out) throws Exception {
                String[] line = in.split(" ");
                for (String s : line) {
                    out.collect(new Tuple2(s, 1));
                }
            }
        })
                .keyBy(0)
                .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
                    @Override
                    public Tuple2<String, Integer> reduce(Tuple2<String, Integer> now, Tuple2<String, Integer> next) throws Exception {
                        return new Tuple2<String, Integer>(now.f0, new Integer(now.f1 + next.f1));
                    }
                })
                .split(new OutputSelector<Tuple2<String, Integer>>() {
                    @Override
                    public Iterable<String> select(Tuple2<String, Integer> in) {
                        if (in.f0.contains("a")) {
                            return Arrays.asList("Contains_a");
                        } else if (in.f0.contains("b")) {
                            return Arrays.asList("Contains_b");
                        } else if (in.f0.contains("c")) {
                            return Arrays.asList("Contains_c");
                        } else {
                            return Arrays.asList("other");
                        }
                    }
                });
        DataStream<Tuple2<String, Integer>> contains_a = streamSplits.select("Contains_a");
        DataStream<Tuple2<String, Integer>> contains_b = streamSplits.select("Contains_b");
        DataStream<Tuple2<String, Integer>> contains_c = streamSplits.select("Contains_c");
        DataStream<Tuple2<String, Integer>> other = streamSplits.select("other");
        //.print("Contains_b");

        ConnectedStreams<Tuple2<String, Integer>, Tuple2<String, Integer>> a_connect_b = contains_a.connect(contains_b);
        //a_connect_b.process ...
        DataStream<Tuple2<String, Integer>> a_union_c = contains_a.union(contains_c);
       // a_union_c.print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

三.window

3.1.time_window

3.1.1.TumblingWindow
package com.flink.window.time_window;
import com.flink.bean.WaterSensor;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
public class TumblingWindow {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //DataStreamSource<String> dsFile = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        //
        //dsFile.setParallelism(1);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        dsWaterSensor
                .keyBy(value->value.getId())
                .timeWindow(Time.seconds(5))
                .reduce((now, next) -> new WaterSensor(now.getId(), next.getTs() + now.getTs(), next.getVc()
                        + now.getVc()))
                .print();
        //dsWaterSensor.print("waterSensor");
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
3.1.2.SlidingWindow
package com.flink.window.time_window;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
public class SlidingWindow {
    public static void main(String[] args) {
        //StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //DataStreamSource<String> dsFile = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        //
        //dsFile.setParallelism(1);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        dsWaterSensor
                .keyBy(value->value.getId())
                .timeWindow(Time.seconds(5),Time.seconds(3))
                .reduce((now, next) -> new WaterSensor(now.getId(), next.getTs() + now.getTs(), next.getVc()
                        + now.getVc()))
                .print();
        //dsWaterSensor.print("waterSensor");
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
3.1.3.SessionWindow
package com.flink.window.time_window;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.assigners.ProcessingTimeSessionWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
public class SessionWindow {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        //
        //dsFile.setParallelism(1);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        dsWaterSensor
                .keyBy(value->value.getId())
                .window(ProcessingTimeSessionWindows.withGap(Time.seconds(3)))
                .reduce((now, next) -> new WaterSensor(now.getId(), next.getTs() + now.getTs(), next.getVc()
                        + now.getVc()))
                .print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

3.2.count_window

3.2.1.TumblingWindow
package com.flink.window.count_window;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
public class TumblingWindow {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //DataStreamSource<String> dsFile = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        //
        //dsFile.setParallelism(1);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        dsWaterSensor
                .keyBy(value->value.getId())
                .countWindow(3)
                .reduce((now, next) -> new WaterSensor(now.getId(), next.getTs() + now.getTs(), next.getVc()
                        + now.getVc()))
                .print();
        //dsWaterSensor.print("waterSensor");
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
3.2.2.SlidingWindow
package com.flink.window.count_window;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
public class SlidingWindow {
    public static void main(String[] args) {
        //StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        //DataStreamSource<String> dsFile = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        //
        //dsFile.setParallelism(1);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        dsWaterSensor
                .keyBy(value->value.getId())
                //没出现两条数据执行一次,执行范围为3条数据    (特殊的:首次出现两条数据直接计算)
                .countWindow(3,2)
                .reduce((now, next) -> new WaterSensor(now.getId(), next.getTs() + now.getTs(), next.getVc()
                        + now.getVc()))
                .print();
        //dsWaterSensor.print("waterSensor");
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

3.3.window_api

//其他窗口API
    //trigger() —— 触发器:定义 window 什么时候关闭，触发计算并输出结果
    //evitor() —— 移除器: 定义移除某些数据的逻辑
    //allowedLateness() —— 允许处理迟到的数据
    //sideOutputLateData() —— 将迟到的数据放入侧输出流
    //getSideOutput() —— 获取侧输出流
3.3.1.incremental_aggregation_functions
//增量聚合函数

IAF_AggregateFunction

package com.flink.window.window_api.incremental_aggregation_functions;
import com.flink.bean.WaterSensor;
import org.apache.flink.api.common.functions.AggregateFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.table.expressions.In;
public class IAF_AggregateFunction {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        //DataStreamSource<String> dsFile = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        //
        //dsFile.setParallelism(1);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        dsWaterSensor
                .keyBy(value->value.getId())
                .timeWindow(Time.seconds(10))
                .aggregate(new AggregateFunction<WaterSensor, Integer, Integer>() {
                    //初始值
                    @Override
                    public Integer createAccumulator() {
                        System.out.println("init...");
                        return 0;
                    }
                    //累加
                    @Override
                    public Integer add(WaterSensor value, Integer accumulator) {
                        System.out.println("add...");
                        return accumulator+1;
                    }
                    //获得结果
                    @Override
                    public Integer getResult(Integer accumulator) {
                        System.out.println("result...");
                        return accumulator;
                    }
                    //会话窗口 才会调用：合并累加器的结果
                    @Override
                    public Integer merge(Integer acc1, Integer acc2) {
                        return acc1+acc2;
                    }
                }).print();

        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

IAF_ReduceFunction

package com.flink.window.window_api.incremental_aggregation_functions;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
public class IAF_ReduceFunction {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //DataStreamSource<String> dsFile = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        //
        //dsFile.setParallelism(1);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        dsWaterSensor
                .keyBy(value->value.getId())
                .timeWindow(Time.seconds(5))
                .reduce((now, next) -> new WaterSensor(now.getId(), next.getTs() + now.getTs(), next.getVc()
                        + now.getVc()))
                .print();
        //dsWaterSensor.print("waterSensor");
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
3.3.2.full_window_functions
//全窗口函数

FWF_ProcessWindowFunction

package com.flink.window.window_api.full_window_functions;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;
public class FWF_ProcessWindowFunction {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        //DataStreamSource<String> dsFile = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        //
        //dsFile.setParallelism(1);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
//升序
        dsWaterSensor
                .keyBy(value->value.getId())
                .timeWindow(Time.seconds(10))
                // 先存数据，等到窗口触发时，才一起计算
                .process(new ProcessWindowFunction<WaterSensor, Long, String, TimeWindow>() {
                    @Override
                    public void process(String key, Context ctx, Iterable<WaterSensor> elements, Collector<Long> out) throws Exception {
                        System.out.println("process");
                        long l = elements.spliterator().estimateSize();
                        out.collect(l);
                    }
                }).print();

        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

四.wartermark

4.1.AccendWatermark

package com.flink.wartermark;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.AscendingTimestampExtractor;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;
//升序
public class AccendWatermark {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.setParallelism(1);
        DataStreamSource<String> dsSocket = env.socketTextStream("localhost",9999);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        SingleOutputStreamOperator<WaterSensor> ds = dsWaterSensor.assignTimestampsAndWatermarks(
                new AscendingTimestampExtractor<WaterSensor>() {
                    @Override
                    public long extractAscendingTimestamp(WaterSensor waterSensor) {
                        return waterSensor.getTs() * 1000L;
                    }
                });

        ds.keyBy(value->value.getId())
                //.window(TumblingEventTimeWindows.of(Time.seconds(5)))
                .timeWindow(Time.seconds(3))
                .process(new ProcessWindowFunction<WaterSensor, String, String, TimeWindow>() {
                    @Override
                    public void process(String key, Context ctx, Iterable<WaterSensor> iterable, Collector<String> out) throws Exception {
                        out.collect("key="+key+
                                "\n"+"window="+ctx.window().getStart()+"--"+ctx.window().getEnd()
                                +"\n"+"watermark="+ctx.currentWatermark()
                                    +"\n"+"size="+iterable.spliterator().estimateSize());
                    }
                })
                .print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

4.2.SequenceWatermark

package com.flink.wartermark;
import com.flink.bean.WaterSensor;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
//乱序
public class SequenceWatermark {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.setParallelism(1);
        DataStreamSource<String> dsSocket = env.socketTextStream("hadoop162",9999);
        SingleOutputStreamOperator<WaterSensor> dsWaterSensor = dsSocket.map(line -> {
            String[] lineValue = line.split(",");
            return new WaterSensor(lineValue[0], Long.valueOf(lineValue[1]), Integer.valueOf(lineValue[2]));
        });
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

五.operator

5.1.O_RollingAggregation_Reduce

package com.flink.operator;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
public class O_RollingAggregation_Reduce {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\word_count.txt");
        ds.flatMap(new FlatMapFunction<String, Tuple2<String,Integer>>() {
            @Override
            public void flatMap(String in, Collector<Tuple2<String,Integer>> out) throws Exception {
                String[] line = in.split(" ");
                for (String s : line) {
                    out.collect(new Tuple2(s,1));
                }
            }
        })
                .keyBy(0)
                //.sum(1)
                .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
                    @Override
                    public Tuple2<String, Integer> reduce(Tuple2<String, Integer> now, Tuple2<String, Integer> next) throws Exception {
                        return new Tuple2<String,Integer>(now.f0,new Integer(now.f1+next.f1));
                    }
                })
                .print();
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

5.2.O_Process

package com.flink.operator;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.util.Collector;
public class O_Process{
    public static class MyProcess extends KeyedProcessFunction<String, Tuple2<String,Integer>,String> {
        @Override
        public void processElement(Tuple2<String, Integer> in, Context ctx, Collector<String> out) throws Exception {
            out.collect("Key: "+ctx.getCurrentKey()+" , "+"Value: "+in);
        }
    }
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\word_count.txt");
        KeyedStream<Tuple2<String, Integer>, String> ks = ds.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String in, Collector<Tuple2<String, Integer>> out) throws Exception {
                String[] line = in.split(" ");
                for (String s : line) {
                    out.collect(new Tuple2(s, 1));
                }
            }
        })
                .keyBy(value -> value.f0);

//                .reduce(new ReduceFunction<Tuple2<String, Integer>>() {
//                    @Override
//                    public Tuple2<String, Integer> reduce(Tuple2<String, Integer> now, Tuple2<String, Integer> next) throws Exception {
//                        return new Tuple2<String,Integer>(now.f0,new Integer(now.f1+next.f1));
//                    }
//                })
                ks.process(new MyProcess())
                .print();

        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

六.sink

6.1.ToKafka

package com.flink.sink;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011;
public class ToKafka {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        ds.addSink(
                new FlinkKafkaProducer011<String>(
                        "hadoop162:9092",
                        "water_sensor",
                        new SimpleStringSchema()
                )
        );
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

6.2.ToES

package com.flink.sink;
import com.flink.bean.WaterSensor;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
import org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink;
import org.apache.flink.table.descriptors.Elasticsearch;
import org.apache.http.HttpHost;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;
import java.util.ArrayList;
import java.util.HashMap;
public class ToES {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        ArrayList<HttpHost> httpHosts = new ArrayList<>();
        httpHosts.add(new HttpHost("hadoop162",9200));
        httpHosts.add(new HttpHost("hadoop163",9200));
        httpHosts.add(new HttpHost("hadoop164",9200));
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        SingleOutputStreamOperator<WaterSensor> dsWS = ds.map(new MapFunction<String, WaterSensor>() {
            @Override
            public WaterSensor map(String in) throws Exception {
                String[] line = in.split(",");
                return new WaterSensor(line[0], Long.valueOf(line[1]), Integer.valueOf(line[2]));
            }
        });
        ElasticsearchSink<String> esSink = new ElasticsearchSink.Builder<String>(
                httpHosts,
                new ElasticsearchSinkFunction<String>() {
                    @Override
                    public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
                        //将数据放入Map
                        HashMap<String, String> dataMap = new HashMap<>();
                        dataMap.put("data", element);
                        //指定索引,类型,source
                        IndexRequest indexRequest = Requests.indexRequest("water_sensor").type("reading").source(dataMap);
                        indexer.add(indexRequest);
                    }
                }
        ).build();
        ds.addSink(esSink);
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

6.3.ToRedis

package com.flink.sink;
import org.apache.commons.math3.fitting.leastsquares.EvaluationRmsChecker;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.redis.RedisSink;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisClusterConfig;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
public class ToRedis {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder()
                .setHost("hadoop162")
                .setPort(6379)
                .build();
        ds.addSink(new RedisSink<String>(
                conf,
                new RedisMapper<String>() {
                    //最外层key
                    @Override
                    public RedisCommandDescription getCommandDescription() {
                        return new RedisCommandDescription(RedisCommand.HSET,"water_sensor");
                    }
                    //hash key
                    @Override
                    public String getKeyFromData(String data) {
                        String[] line= data.split(",");
                        return line[0]+line[1];
                    }
                    //hash value
                    @Override
                    public String getValueFromData(String data) {
                        String[] line= data.split(",");
                        return line[2];
                    }
                }
        ));
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

6.4.CustomizeSink

package com.flink.sink;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
public class CustomizeSink {
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> ds = env.readTextFile("D:\\code\\flink111\\src\\main\\resources\\input\\water_sensor.txt");
        ds.addSink(
                new RichSinkFunction<String>() {
                    private Connection conn=null;
                    private PreparedStatement ps=null;
                    @Override
                    public void open(Configuration parameters) throws Exception {
                        conn= DriverManager.getConnection("jdbc:postgresql://192.168.1.163:5432/test_db2","test_user","aaaaaa");
                        ps=conn.prepareStatement("INSERT INTO water_sensor values(?,?,?)");
                    }
                    @Override
                    public void invoke(String value, Context ctx) throws Exception {
                        String[] line = value.split(",");
                        ps.setString(1,line[0]);
                        ps.setLong(2,Long.valueOf(line[1]));
                        ps.setInt(3,Integer.valueOf(line[2]));
                        ps.execute();
                    }
                    @Override
                    public void close() throws Exception {
                        ps.close();
                        conn.close();
                    }


                }

        );
        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

七.process_function(待完善)

Flink提供了8个Process Function：
    ProcessFunction
    KeyedProcessFunction
    CoProcessFunction
    ProcessJoinFunction
    BroadcastProcessFunction
    KeyedBroadcastProcessFunction
    ProcessWindowFunction
    ProcessAllWindowFunction
//操作上下文环境获取 窗口信息,watermark信息 当前key 等......

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.flink</groupId>
    <artifactId>flink1.11</artifactId>
    <version>1.0-SNAPSHOT</version>
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <flink.version>1.11.2</flink.version>
        <java.version>1.8</java.version>
        <scala.binary.version>2.11</scala.binary.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
        <log4j.version>2.12.1</log4j.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/log4j/log4j -->
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-shaded-hadoop-2-uber -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-shaded-hadoop-2-uber</artifactId>
            <version>2.7.5-7.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka-0.11_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
        <dependency>
            <groupId>org.postgresql</groupId>
            <artifactId>postgresql</artifactId>
            <version>42.2.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_2.11</artifactId>
            <version>${flink.version}</version>
            <!--            <classifier>sources</classifier>-->
            <!--            <type>java-source</type>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-common</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.bahir/flink-connector-redis -->
        <dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_${scala.binary.version}</artifactId>
            <version>1.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch6_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
        <dependency>
            <groupId>org.postgresql</groupId>
            <artifactId>postgresql</artifactId>
            <version>42.2.5</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.0.0</version>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

bean.WaterSensor

WaterSensor

package com.flink.bean;
// 定义样例类：水位传感器：用于接收空高数据
// id:传感器编号
// ts:时间戳
// vc:空高
public class WaterSensor {
    private String id;
    private Long ts;
    private Integer vc;

    public WaterSensor() {
    }

    public WaterSensor(String id, Long ts, Integer vc) {
        this.id = id;
        this.ts = ts;
        this.vc = vc;
    }

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public Long getTs() {
        return ts;
    }

    public void setTs(Long ts) {
        this.ts = ts;
    }

    public Integer getVc() {
        return vc;
    }

    public void setVc(Integer vc) {
        this.vc = vc;
    }

    @Override
    public String toString() {
        return "WaterSensor{" +
                "id='" + id + '\'' +
                ", ts=" + ts +
                ", vc=" + vc +
                '}';
    }
}

input

water_sensor.txt

sensor_1,1549044122,1
sensor_1,1549044123,2
sensor_1,1549044124,3
sensor_1,1549044125,4
sensor_2,1549044123,2
sensor_3,1549044124,3
sensor_4,1549044125,4
sensor_5,1549044126,5
sensor_6,1549044127,6
sensor_7,1549044128,7
sensor_8,1549044129,8
sensor_9,1549044130,9
sensor_101549044130,10
sensor_11,1549044122,11
sensor_12,1549044123,12
sensor_13,1549044124,13
sensor_14,1549044125,14
sensor_15,1549044126,15
sensor_16,1549044127,16
sensor_17,1549044128,17
sensor_18,1549044129,18
sensor_19,1549044130,19

word_count.txt

a1 a2 b1
b1 b2 c1 c2 a2
c2 b2 b3
b1 b2 c3 c2 a1
c2 b2 b3
b1 b1 c3 c2 a1
c2 b1 b2
稻草人的肩膀
关注
专栏目录
flink入门部署demo
网维的木棉后记
11-03 228
flink 本地部署 wget https://mirrors.bfsu.edu.cn/apache/flink/flink-1.11.2/flink-1.11.2-bin-scala_2.11.tgz [hadoop@slave1 soft]$ tar -zxvf flink-1.11.2-bin-scala_2.11.tgz [hadoop@slave1 soft]$ cd flink-1.11.2 [hadoop@slave1 flink-1.11.2]$ pwd /home/hadoop/sof
flink-1.11.2-bin-scala_2.11.tgz
11-03
这个压缩包“flink-1.11.2-bin-scala_2.11.tgz”包含了Flink运行所需的所有核心组件和库，支持Scala 2.11版本。解压后，用户可以直接在本地环境中启动和运行Flink作业。 1. **Flink核心概念**： - **流处理**：...
flink-1.11.2-bin-scala_2.12 linux安装包
10-14
flink-1.11.2-bin-scala_2.12 linux安装包
flink source code
weixin_30505751的博客
11-05 59
https://github.com/apache/flink/tree/master/docs https://github.com/flink-china/1.6.0 https://github.com/alibaba/flink 转载于:https://www.cnblogs.com/WCFGROUP/p/9906759.html
java Flink 1.11 demo
Aspirin's Nest
11-19 481
其实我挺排斥用java写大数据代码的，但是没办法公司用java写。不过Java写也是有道理的嘛，所以就从Scala转到了Java。 依赖 <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-java</artifactId> <version>${flink.version}</version> <!--
flink-statebackend-rocksdb_2.11-1.13.2-API文档-中文版.zip
06-06
赠送jar包：flink-statebackend-rocksdb_2.11-1.13.2.jar； 赠送原API文档：flink-statebackend-rocksdb_2.11-1.13.2-javadoc.jar； 赠送源代码：flink-statebackend-rocksdb_2.11-1.13.2-sources.jar； 赠送Maven...
flink-connector-jdbc_2.11-1.11.2.jar
02-24
flink本身DDL的方式目前还不支持clickhouse，本jar添加对clickhouse的支持，导入到flink/lib下即可使用
flink-sql-connector-kafka_2.11-1.11.2.jar
03-12
flink-sql-connector-kafka_2.11-1.11.2.jar
Flink（十）【处理函数】
功不唐捐，玉汝于成
12-20 1370
Flink 处理函数
Flink基础
Samooyou的博客
03-06 1236
目录 Flink自定义水印 Flink定时器Timer Flink自定义水印 package com.home.flink.chapter07; import com.home.flink.bean.WaterSensor; import com.home.flink.util.homeUtil; import org.apache.flink.api.common.eventtime.*; import org.apache.flink.configuration.Configu
【FLink】水位线（Watermark）
lxtx
11-21 3684
在Flink中，用来衡量事件时间进展的标记，就被称作“水位线”（Watermark）。说白了就是事件时间戳。
Flink官方 code walkthroughs 踩坑
weixin_43432984的博客
02-29 2563
Flink官方 code walkthroughs 踩坑 直接运行项目中的FraudDetectionJob会出现以下错误 Exception in thread “main” java.lang.NoClassDefFoundError: org/apache/flink/streaming/api/functions/source/SourceFunction at spendreport.F...
【Flink】【第五章 流处理API】Source 和 Sink
weixin_43589563的博客
01-19 908
Source 输入Controlling Latency(控制延迟) Source传输控制策略_29集 默认情况下，流中的元素不会一个一个的在网络中传输，而是缓存起来，这样可以减少连接的次数，提高效率； 缓存的大小可以在Flink的配置文件、ExecutionEnvironment、或者某个算子上进行配置； 输入控制延迟默认为100ms 输入延迟的好处：提高吞吐量 输入延迟的缺点：增加了延迟 如何把握平衡？ 如果是为了最大吞吐量，则可以设置setBufferTimeout(-1)，移除timeout
狗屎运！居然成为了Flink的贡献者-如何给开源项目贡献代码
eeeddieee的博客
04-08 325
记录了一个给开源软件项目贡献代码的通用过程。
Flink结合Kafka实时写入Iceberg实践笔记
欲望以提升热忱 毅力以磨平高山
01-19 3406
前言 上文提到使用Flink SQL写入hadoop catalog 的iceberg table 的简单示例，这次我就flink 消费kafka 流式写入iceberg table做一个验证，现记录如下： 环境：本地测试环境 JDK1.8 、Flink 1.11.2 、Hadoop3.0.0 、Hive2.1.1 一、前置说明 本文记录了使用HDFS的一个路径作为iceberg 的结果表，使用Flink实时消费kafka中的数据并写入iceberg表，并且使用Hive作为客户端实时读取。 因为..
flink从kafka读数据demo(java)
仙道Bob的专栏
11-22 1305
1.结构 flink-demo --src --main --java --com.bob.demo --App --DemoRichSink --FlinkDemo --resources --consumer.properties --start.sh --pom.xml 2.具体代码 2.1 pom.xml <?xml version="1.0" encoding="UTF-8
实时计算Flink
code__online的博客
05-02 974
实时计算Flink 三种实时计算框架storm、spark streaming和flink的对比 storm延迟低但吞吐量小 spark streaming吞吐量大，但延迟高 flink是一种兼具低延迟和高吞吐量特点的流计算技术，还是一套框架中能同时支持批处理和流处理的 一个计算平台 Flink流处理特性 高吞吐、低延迟、高性能 支持带有事件时间的窗口（Window）操作 支持有状态计算的Exactly-once语义 支持高度灵活的窗口（Window）操作，支持基于time、count、session
flink1.11.0基于java代码实现Application模式的踩坑总结
博客
08-04 1737
一、SubmitJobApplicationMode 1、将core-site.xml,flink-conf.yaml,hdfs-site.xml,mapred-site.xml,yarn-site.xml放到resources目录下 2、无法读取到本地的conf文件 org.apache.hadoop.conf.Configuration conf = new org.apache.conf.Configuration(); conf.set("fs.hdfs.impl","org.apache.
flink-streaming-java_2.11
最新发布
02-02
flink-streaming-java_2.11是Apache Flink项目的一个模块，用于开发和执行流式处理任务。它使用Java语言编写，并支持Scala和Python等其他语言的API。 Apache Flink是一个分布式流式处理框架，支持可靠和高性能的...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值