峰谷均计算

package com.mz.iot;

import com.alibaba.fastjson.JSON;
import com.mz.iot.bean.MaxMinAvg;
import com.mz.iot.bean.DataOrigin;
import com.mz.iot.utils.Conf;
import com.mz.iot.utils.DateUtil;
import com.mz.iot.utils.EsSinkUtil;
import com.mz.iot.utils.JsonUtil;
import org.apache.flink.api.common.functions.AggregateFunction;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.xcontent.XContentType;

/**
 * 所有设备类型的峰值谷值平均值统计
 * 窗口有15min，1...24小时
 */
public class MaxMinAvgAnalysis {
    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        env.setParallelism(1);

        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

        /**
         * 检查点配置
         */
        env.enableCheckpointing(1000, CheckpointingMode.EXACTLY_ONCE);
        env.getCheckpointConfig().setCheckpointTimeout(60000L);
        env.getCheckpointConfig().setMaxConcurrentCheckpoints(2);
        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(100L);
        env.getCheckpointConfig().setPreferCheckpointForRecovery(true);
        env.getCheckpointConfig().setTolerableCheckpointFailureNumber(1);

        env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);


        /**
         * 重启策略配置,默认，不停的重启
         */
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(0, 3000L));

        DataStream<String> inputStream = env.socketTextStream("192.168.0.162", 6666).uid("mma-input");

        SingleOutputStreamOperator<DataOrigin> dataStream = inputStream.map(new MapFunction<String, DataOrigin>() {
            @Override
            public DataOrigin map(String value) throws Exception {

                DataOrigin dataOrigin = JsonUtil.getObjFromJson(value, DataOrigin.class);

//                System.out.println("useOrigin:" + dataOrigin);

                if (dataOrigin == null) {
                    DataOrigin dataOrigin1 = new DataOrigin();
                    dataOrigin1.setId("invalid");
                    return dataOrigin1;
                }

                return dataOrigin;

            }
        })
                .uid("mma-origin-map")
                .filter(new FilterFunction<DataOrigin>() {
                    @Override
                    public boolean filter(DataOrigin value) throws Exception {
                        return !"invalid".equals(value.getId());
                    }
                })
                .uid("mma-filter")
                .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<DataOrigin>(Time.seconds(5)) {
                    @Override
                    public long extractTimestamp(DataOrigin element) {

                        String acquisitionTime = element.getAcquisitionTime();

                        return DateUtil.getMillsFromString(acquisitionTime);
                    }
                })
                .uid("mma-assign-wm");

        /**
         * 峰值谷值均值相关的数据
         */
        OutputTag<DataOrigin> f_main = new OutputTag<DataOrigin>("f0") {
        };

        /**
         * 迟到数据
         */
        OutputTag<DataOrigin> tag_late = new OutputTag<DataOrigin>("late") {

        };

        SingleOutputStreamOperator<DataOrigin> process = dataStream.process(new ProcessFunction<DataOrigin, DataOrigin>() {
            @Override
            public void processElement(DataOrigin value, Context ctx, Collector<DataOrigin> out) throws Exception {

                if ("0".equals(value.getMakeUp())) {

                    if ("0".equals(value.getMeterKind()) && "4".equals(value.getType()) || "1".equals(value.getMeterKind())) {

                    } else {
                        ctx.output(f_main, value);
                    }
                }
            }
        });

        DataStream<DataOrigin> stream_main = process.getSideOutput(f_main);

        stream_main.print("max_min_avg_stream_main====>");


        SingleOutputStreamOperator<MaxMinAvg> windowResult_15min = stream_main
                .keyBy("projectId", "meterKind", "meterCode", "meterVarId", "type", "name")
                .window(TumblingEventTimeWindows.of(Time.minutes(15), Time.minutes(0)))
                .allowedLateness(Time.minutes(1))
                .sideOutputLateData(tag_late)
                .aggregate(new MAxMinAvgAgg(), new MaxMinAvgWindowResult("9999"));
        windowResult_15min.print("windowResult_15min=====>");

        DataStream<DataOrigin> stream_late = windowResult_15min.getSideOutput(tag_late);

        stream_late.print("streamLate======>");


//        SingleOutputStreamOperator<MaxMinAvg> aggregateStream_1hour = f2_stream
//                .keyBy("projectId", "meterKind", "meterCode", "meterVarId", "type", "name")
//                .window(TumblingEventTimeWindows.of(Time.hours(1), Time.minutes(0)))
//                .aggregate(new MAxMinAvgAgg(), new MaxMinAvgWindowResult("1"));
//        aggregateStream_1hour.print("aggregateStream_1hour=====>");


        int bulkSize = 40;
        int sinkParallelism = 5;

        EsSinkUtil.addSink("192.168.0.163:9200", bulkSize, sinkParallelism, windowResult_15min, new EsSinkResult(Conf.iot_ads_x_hour), "mma-sink-result-15min");
        EsSinkUtil.addSink("192.168.0.163:9200", bulkSize, sinkParallelism, windowResult_15min, new EsSinkResult(Conf.iot_ads_x_hour_9999), "mma-sink-result");
        EsSinkUtil.addSink("192.168.0.163:9200", bulkSize, sinkParallelism, stream_late, new EsSinkLate(), "mma-sink-late");


        env.execute("max min avg analysis");

    }

    public static class EsSinkResult implements ElasticsearchSinkFunction<MaxMinAvg> {

        private final String indexName;

        public EsSinkResult(String indexName) {
            this.indexName = indexName;
        }

        @Override
        public void process(MaxMinAvg element, RuntimeContext ctx, RequestIndexer indexer) {

            IndexRequest indexRequest = Requests.indexRequest()
                    .index(indexName)
                    .type("_doc")
                    .id(element.getPk() + "")
                    .source(JSON.toJSONString(element), XContentType.JSON);

            indexer.add(indexRequest);
        }
    }

    public static class EsSinkLate implements ElasticsearchSinkFunction<DataOrigin> {

        @Override
        public void process(DataOrigin element, RuntimeContext ctx, RequestIndexer indexer) {

            System.out.println("sink late...");

            IndexRequest indexRequest = Requests.indexRequest()
                    .index(Conf.iot_big_data_mma_late_index)
                    .type("_doc")
                    .id(element.getId() + "")
                    .source(JSON.toJSONString(element), XContentType.JSON);

            indexer.add(indexRequest);
        }
    }


    /**
     * exec when every elements comes
     */
    public static class MAxMinAvgAgg implements AggregateFunction<DataOrigin, MaxMinAvg, MaxMinAvg> {

        @Override
        public MaxMinAvg createAccumulator() {

            return new MaxMinAvg();
        }

        @Override
        public MaxMinAvg add(DataOrigin value, MaxMinAvg accumulator) {


            float max = Math.max(accumulator.getMax_dataValue(), value.getAddValue());

            float min = Math.min(accumulator.getMin_dataValue(), value.getAddValue());

            float sum = accumulator.getTotal_cnts() + value.getAddValue();

            long counts = accumulator.getFrequency() + 1;

            float avg = sum / counts;

            accumulator.setMax_dataValue(max);
            accumulator.setMin_dataValue(min);
            accumulator.setTotal_cnts(sum);
            accumulator.setFrequency(counts);
            accumulator.setAvg(avg);

            System.out.println("accumulator:" + accumulator);
            return accumulator;
        }

        @Override
        public MaxMinAvg getResult(MaxMinAvg accumulator) {
            return accumulator;
        }

        @Override
        public MaxMinAvg merge(MaxMinAvg a, MaxMinAvg b) {
            return null;
        }
    }

    public static class MaxMinAvgWindowResult extends ProcessWindowFunction<MaxMinAvg, MaxMinAvg, Tuple, TimeWindow> {

        private final String span;

        public MaxMinAvgWindowResult(String span) {
            this.span = span;
        }


        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            System.out.println("window function open");
        }


        @Override
        public void close() throws Exception {
            super.close();
            System.out.println("window function close");
        }

        @Override
        public void process(Tuple tuple, Context context, Iterable<MaxMinAvg> elements, Collector<MaxMinAvg> out) throws Exception {


            MaxMinAvg e = elements.iterator().next();

            System.out.println("e:" + e);


            String t_start = DateUtil.getDateStrFromMill(context.window().getStart());

            String t_end = DateUtil.getDateStrFromMill(context.window().getEnd());
            String wm = DateUtil.getDateStrFromMill(context.currentWatermark());


            System.out.println("window process,t_start:" + t_start + ",t_end:" + t_end + ",watermark:" + wm);

            String dt = t_start.split(" ")[0];
            String dh = t_start.split(" ")[1].split(":")[0];
            String dm = t_start.split(" ")[1].split(":")[1];

            /**
             , concat_ws(""
             , col("dt")
             , col("dh")
             , col("dm")
             , col("projectIdentify")
             , col("meterKind")
             , col("equipmentNo")
             , col("meterVarId")
             , col("type")
             , col("name")
             , col("span")
             */
            /**
             *  .keyBy("projectId", "meterKind", "meterCode", "meterVarId", "type", "name")
             */

            String projectId = tuple.getField(0);
            String meterKind = tuple.getField(1);
            String meterCode = tuple.getField(2);
            String meterVarId = tuple.getField(3);
            String type = tuple.getField(4);
            String name = tuple.getField(5);


            e.setDt(dt);
            e.setDh(dh);
            e.setDm(dm);
            e.setT_start(t_start);
            e.setProjectIdentify(projectId);
            e.setMeterKind(meterKind);
            e.setEquipmentNo(meterCode);
            e.setMeterVarId(meterVarId);
            e.setType(type);
            e.setName(name);
            e.setSpan(span);

            e.setPk(dt + dh + dm + projectId + meterKind + meterCode + meterVarId + type + name + span);

            e.setLast_update_time(DateUtil.getDateStrFromMill(context.currentProcessingTime()));

            out.collect(e);
        }

    }
}