Flink使用自定义Source Sink reduce aggregate统计最小值及平均值

模拟设备上报温度, 获取最低温度

pom.xml

<dependencies>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-java</artifactId>
        <version>1.12.4</version>
    </dependency>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-streaming-java_2.12</artifactId>
        <version>1.12.4</version>
    </dependency>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-clients_2.12</artifactId>
        <version>1.12.4</version>
    </dependency>
</dependencies>

使用无界数据, flatMap(过滤转换), keyBy(分组), window(窗口), reduce等函数实现最小值统计

package com.demo.flink;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.datastream.WindowedStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.util.Collector;

import java.io.Serializable;
import java.math.BigDecimal;
import java.util.Date;
import java.util.Random;

public class FlinkTest {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // 使用自定义数据源
        DataStream<DeviceInfo> dataStream = env.addSource(new MyDataSource());
        // 使用flatMap过滤 转换
        SingleOutputStreamOperator<DeviceInfo> objectSingleOutputStreamOperator = dataStream.flatMap(new FlatMapFunction<DeviceInfo, DeviceInfo>() {
            @Override
            public void flatMap(DeviceInfo deviceInfo, Collector<DeviceInfo> collector) throws Exception {
                // 可以过滤 并且转换  过滤DEV_00004设备
                if (!"DEV_00004".equals(deviceInfo.getDeviceId())) {
                    collector.collect(deviceInfo);
                }
            }
        });
        // 使用keyBy对数据进行分组  使用deviceId字段作为Key
        KeyedStream<DeviceInfo, String> deviceInfoStringKeyedStream = objectSingleOutputStreamOperator.keyBy(new KeySelector<DeviceInfo, String>() {
            @Override
            public String getKey(DeviceInfo deviceInfo) {
                return deviceInfo.getDeviceId();
            }
        });
        // 滚动处理窗口 5秒一个窗口
        WindowedStream<DeviceInfo, String, TimeWindow> window = deviceInfoStringKeyedStream
                .window(TumblingProcessingTimeWindows.of(Time.seconds(5), Time.seconds(4)));
        SingleOutputStreamOperator<DeviceInfo> reduce = window.reduce(new ReduceFunction<DeviceInfo>() {
            @Override
            public DeviceInfo reduce(DeviceInfo t1, DeviceInfo t2) throws Exception {
            	// 两个数据做比较 返回最小值
                return new DeviceInfo(t1.getDeviceId(), t1.getTemperature().compareTo(t2.getTemperature()) > 0 ? t2.getTemperature() : t1.getTemperature());
            }
        });
        // 自定义输出
        reduce.addSink(new MySink());
        env.execute("My flink test");
    }

}

class DeviceInfo implements Serializable {
    private String deviceId;
    private BigDecimal temperature;

    public DeviceInfo(String deviceId, BigDecimal temperature) {
        this.deviceId = deviceId;
        this.temperature = temperature;
    }

    public String getDeviceId() { return deviceId; }

    public void setDeviceId(String deviceId) { this.deviceId = deviceId; }

    public BigDecimal getTemperature() { return temperature; }

    public void setTemperature(BigDecimal temperature) { this.temperature = temperature; }

    @Override
    public String toString() {
        return "DeviceInfo{" +
                "deviceId='" + deviceId + '\'' +
                ", temperature=" + temperature +
                '}';
    }
}

class MyDataSource implements SourceFunction<DeviceInfo> {

    private volatile boolean running = true;

    private String[] deviceIds = new String[]{"DEV_00001", "DEV_00002", "DEV_00003", "DEV_00004", "DEV_00005", "DEV_00006"};

    @Override
    public void run(SourceContext<DeviceInfo> ctx) throws Exception {
        while (running) {
            ctx.collect(new DeviceInfo(
                    deviceIds[new Random().nextInt(deviceIds.length)],
                    new BigDecimal(new Random().nextFloat() * 30.0f).setScale(2, BigDecimal.ROUND_DOWN)));
            Thread.sleep(200);
        }
    }

    @Override
    public void cancel() {
        running = false;
    }
}

class MySink extends RichSinkFunction<DeviceInfo> {

    @Override
    public void invoke(DeviceInfo value, Context context) throws Exception {
        System.out.println(
                "ID: " + value.getDeviceId()
                        + ", temperature: " + value.getTemperature()
                        + ", time: " + context.timestamp()
                        + ", now: " + new Date().getTime());
    }

}

运行结果

使用aggregate函数求平均数

自定义对象用来保存总记录数和总数, 结果对象保存平均温度.

package com.demo.flink;

import org.apache.flink.api.common.functions.AggregateFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;

import java.io.Serializable;
import java.math.BigDecimal;
import java.util.Date;
import java.util.Random;

public class AvgTest {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStream<DeviceInfo> dataStream = env.addSource(new MyDataSource());
        // keyBy分流 使用deviceId作为Key
        dataStream.keyBy(new KeySelector<DeviceInfo, String>() {
            @Override
            public String getKey(DeviceInfo deviceInfo) throws Exception {
                return deviceInfo.getDeviceId();
            }
        }).window(TumblingProcessingTimeWindows.of(Time.seconds(5))) // 每5s滚动开窗
                // 1.数据源对象
                // 2.累加后对象
                // 3.平均后对象
                .aggregate(new AggregateFunction<DeviceInfo, AggregateResult, AvgResult>() {
                    @Override
                    public AggregateResult createAccumulator() {
                        return new AggregateResult("", 0, BigDecimal.ZERO);
                    }

                    @Override
                    public AggregateResult add(DeviceInfo deviceInfo, AggregateResult aggregateResult) {
                        aggregateResult.setDeviceId(deviceInfo.getDeviceId());
                        aggregateResult.setCount(aggregateResult.getCount() + 1);
                        aggregateResult.setSumTemperature(aggregateResult.getSumTemperature().add(deviceInfo.getTemperature()));
                        return aggregateResult;
                    }

                    @Override
                    public AvgResult getResult(AggregateResult aggregateResult) {
                        AvgResult result = new AvgResult();
                        result.setDeviceId(aggregateResult.getDeviceId());
                        result.setAvgTemperature(aggregateResult.getSumTemperature().divide(BigDecimal.valueOf(aggregateResult.getCount()), 2, BigDecimal.ROUND_HALF_UP));
                        return result;
                    }

                    @Override
                    public AggregateResult merge(AggregateResult aggregateResult, AggregateResult acc1) {
                        aggregateResult.setCount(aggregateResult.getCount() + acc1.getCount());
                        aggregateResult.setSumTemperature(aggregateResult.getSumTemperature().add(acc1.getSumTemperature()));
                        return aggregateResult;
                    }
                }).addSink(new MySink());
        env.execute("My flink test");
    }

}

class DeviceInfo implements Serializable {
    private String deviceId;
    private BigDecimal temperature;

    public DeviceInfo(String deviceId, BigDecimal temperature) {
        this.deviceId = deviceId;
        this.temperature = temperature;
    }

    public String getDeviceId() { return deviceId; }
    public void setDeviceId(String deviceId) { this.deviceId = deviceId; }
    public BigDecimal getTemperature() { return temperature; }
    public void setTemperature(BigDecimal temperature) { this.temperature = temperature; }
}

class MyDataSource implements SourceFunction<DeviceInfo> {

    private volatile boolean running = true;

    private String[] deviceIds = new String[]{"DEV_00001", "DEV_00002", "DEV_00003", "DEV_00004", "DEV_00005", "DEV_00006"};

    @Override
    public void run(SourceContext<DeviceInfo> ctx) throws Exception {
        while (running) {
            ctx.collect(new DeviceInfo(
                    deviceIds[new Random().nextInt(deviceIds.length)],
                    new BigDecimal(new Random().nextFloat() * 30.0f).setScale(2, BigDecimal.ROUND_DOWN)));
            Thread.sleep(200);
        }
    }

    @Override
    public void cancel() {
        running = false;
    }
}

class AggregateResult implements Serializable {
    private String deviceId;
    private long count;
    private BigDecimal sumTemperature;

    public AggregateResult(String deviceId, long count, BigDecimal sumTemperature) {
        this.deviceId = deviceId;
        this.count = count;
        this.sumTemperature = sumTemperature;
    }

    public String getDeviceId() { return deviceId; }
    public void setDeviceId(String deviceId) { this.deviceId = deviceId; }
    public long getCount() { return count; }
    public void setCount(long count) { this.count = count; }
    public BigDecimal getSumTemperature() { return sumTemperature; }
    public void setSumTemperature(BigDecimal sumTemperature) { this.sumTemperature = sumTemperature; }
}

class AvgResult implements Serializable {
    private String deviceId;
    private BigDecimal avgTemperature;

    public String getDeviceId() { return deviceId; }
    public void setDeviceId(String deviceId) { this.deviceId = deviceId; }
    public BigDecimal getAvgTemperature() { return avgTemperature; }
    public void setAvgTemperature(BigDecimal avgTemperature) { this.avgTemperature = avgTemperature; }
}

class MySink extends RichSinkFunction<AvgResult> {

    @Override
    public void invoke(AvgResult value, Context context) throws Exception {
        System.out.println(
                "ID: " + value.getDeviceId()
                        + ", temperature: " + value.getAvgTemperature()
                        + ", time: " + context.timestamp()
                        + ", now: " + new Date().getTime());
    }

}

运行结果