Flink1.11.2-1Json-sideProcess

Flink1.11.2-1Json-sideProcess

ToHdfs

package com.flink.app.toHdfs;

import com.flink.app.parJson.P_PageLog;
import com.flink.app.parJson.P_PageLog2;
import com.flink.bean.PageLog;
import com.flink.bean.PageLog2;
import com.hdfs.HdfsUtils;
import com.kafka.kafka_api.KafkaUtils;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import org.apache.flink.util.OutputTag;

public class ToHdfs {
    public static void main(String[] args) {
        // 创建执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        FlinkKafkaConsumer011 kafkaSource = KafkaUtils.getFlinkKafkaConsumer("pageLog");
        DataStreamSource<String> dsJsonStr = env.addSource(kafkaSource);
        SingleOutputStreamOperator<String> processDS = dsJsonStr.process(new MySideProcess());
        OutputTag<String> pageLogOutputTag = new OutputTag<String>("pageLog"){};
        //pageLog sink
        processDS.getSideOutput(pageLogOutputTag)
                .map(new MapFunction<String, PageLog>() {
                    @Override
                    public PageLog map(String in) throws Exception {
                        PageLog pageLog = P_PageLog.getPageLog(in);
                        System.out.println(pageLog);
                        return pageLog;
                    }
                })
//                .print("pageLog")
                .addSink(HdfsUtils.getPageLogBucketingSink("hdfs://192.168.1.162:8020/kafkaTohdfs/pageLog"));

        //pageLog2 sink
        OutputTag<String> pageLog2OutputTag = new OutputTag<String>("pageLog2"){};
        processDS.getSideOutput(pageLog2OutputTag)
                .map(new MapFunction<String, PageLog2>() {
                    @Override
                    public PageLog2 map(String in) throws Exception {
                        PageLog2 pageLog2 = P_PageLog2.getPageLog2(in);
                        System.out.println(pageLog2);
                        return pageLog2;
                    }
                })
//                .print("pageLog2")
                .addSink(HdfsUtils.getPageLog2BucketingSink("hdfs://192.168.1.162:8020/kafkaTohdfs/pageLog2/"));
        // BucketingSink sink = HdfsUtils.getBucketingSink("hdfs://192.168.1.162:8020/kafkaTohdfs/");
        //dsStr.addSink(sink);




        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


}

MySideProcess

package com.flink.app.toHdfs;

import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

public class MySideProcess extends ProcessFunction<String,String> {
    OutputTag<String> pageLog = new OutputTag<String>("pageLog"){};
    OutputTag<String> pageLog2 = new OutputTag<String>("pageLog2"){};
    @Override
    public void processElement(String value, Context ctx, Collector<String> out) throws Exception {
        ctx.output(pageLog,value);
        ctx.output(pageLog2,value);
        out.collect(value);
    }
}

HdfsUtils

package com.hdfs;

import com.flink.bean.PageLog;
import com.flink.bean.PageLog2;
import org.apache.flink.streaming.connectors.fs.StringWriter;
import org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink;

public class HdfsUtils {
    public static  BucketingSink getPageLogBucketingSink(String basePath ){
        //basePath="hdfs://192.168.1.162:8020/kafkaTohdfs/"
        BucketingSink<PageLog> sink = new BucketingSink<>(basePath);
//通过这样的方式来实现数据跨天分区
//        sink.setBucketer(new EventTimeBucketer<PageLog>("yyyy/MM/dd"));
        sink.setBucketer(new PageLogEventTimeBucketer("yyyy-MM-dd/HH"));
        sink.setWriter(new StringWriter<>());
        // sink.setBatchSize(1024 * 1024 * 256L);
        sink.setBatchRolloverInterval(1 * 60 * 1000L);   //时间
        //sink.setInactiveBucketThreshold(3 * 60 * 1000L);
        //sink.setInactiveBucketCheckInterval(30 * 1000L);
        //设置的是检查两次检查桶不活跃的情况的周期
        sink.setInactiveBucketCheckInterval(1*60*1000L);
        //设置的是关闭不活跃桶的阈值,多久时间没有数据写入就关闭桶
        sink.setInactiveBucketThreshold(1*60*1000L);

        sink.setInProgressSuffix(".in-progress");
        sink.setPendingSuffix(".pending");
        return sink;
    }
    public static  BucketingSink getPageLog2BucketingSink(String basePath ){
        //basePath="hdfs://192.168.1.162:8020/kafkaTohdfs/"
        BucketingSink<PageLog2> sink = new BucketingSink<>(basePath);
//通过这样的方式来实现数据跨天分区
//        sink.setBucketer(new EventTimeBucketer<PageLog>("yyyy/MM/dd"));
        sink.setBucketer(new PageLog2EventTimeBucketer("yyyy-MM-dd/HH"));
        sink.setWriter(new StringWriter<>());
        // sink.setBatchSize(1024 * 1024 * 256L);
        sink.setBatchRolloverInterval(1 * 60 * 1000L);   //时间
        //sink.setInactiveBucketThreshold(3 * 60 * 1000L);
        //sink.setInactiveBucketCheckInterval(30 * 1000L);
        //设置的是检查两次检查桶不活跃的情况的周期
        sink.setInactiveBucketCheckInterval(1*60*1000L);
        //设置的是关闭不活跃桶的阈值,多久时间没有数据写入就关闭桶
        sink.setInactiveBucketThreshold(1*60*1000L);

        sink.setInProgressSuffix(".in-progress");
        sink.setPendingSuffix(".pending");
        return sink;
    }
}

PageLog2EventTimeBucketer

package com.hdfs;
import com.flink.bean.PageLog2;
import org.apache.flink.streaming.connectors.fs.Clock;
import org.apache.flink.streaming.connectors.fs.bucketing.Bucketer;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.io.ObjectInputStream;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;

public class PageLog2EventTimeBucketer implements Bucketer<PageLog2> {
    private static final String DEFAULT_FORMAT_STRING = "yyyy/MM/dd";
    private final String formatString;
    private final ZoneId zoneId;
    private transient DateTimeFormatter dateTimeFormatter;
    public PageLog2EventTimeBucketer() {
        this(DEFAULT_FORMAT_STRING);
    }
    public PageLog2EventTimeBucketer(String formatString) {
        this(formatString, ZoneId.systemDefault());
    }
    public PageLog2EventTimeBucketer(ZoneId zoneId) {
        this(DEFAULT_FORMAT_STRING, zoneId);
    }
    public PageLog2EventTimeBucketer(String formatString, ZoneId zoneId) {
        this.formatString = formatString;
        this.zoneId = zoneId;
        this.dateTimeFormatter = DateTimeFormatter.ofPattern(this.formatString).withZone(this.zoneId);
    }
    //记住，这个方法一定要加，否则dateTimeFormatter对象会是空，此方法会在反序列的时候调用，这样才能正确初始化dateTimeFormatter对象
    //那有的人问了，上面构造函数不是初始化了吗？反序列化的时候是不走构造函数的
    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
        in.defaultReadObject();
        this.dateTimeFormatter = DateTimeFormatter.ofPattern(formatString).withZone(zoneId);
    }
    @Override
    public Path getBucketPath(Clock clock, Path basePath, PageLog2 element) {
        String newDateTimeString = dateTimeFormatter.format(Instant.ofEpochMilli(element.getuId()*1000));
      // System.out.println(element.getuId());
       // System.out.println(newDateTimeString);
        return new Path(basePath + "/" + newDateTimeString);
    }
}

PageLogEventTimeBucketer

package com.hdfs;
import com.flink.bean.PageLog;
import org.apache.flink.streaming.connectors.fs.Clock;
import org.apache.flink.streaming.connectors.fs.bucketing.Bucketer;
import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;

public class PageLogEventTimeBucketer implements Bucketer<PageLog> {
    private static final String DEFAULT_FORMAT_STRING = "yyyy/MM/dd";
    private final String formatString;
    private final ZoneId zoneId;
    private transient DateTimeFormatter dateTimeFormatter;
    public PageLogEventTimeBucketer() {
        this(DEFAULT_FORMAT_STRING);
    }
    public PageLogEventTimeBucketer(String formatString) {
        this(formatString, ZoneId.systemDefault());
    }
    public PageLogEventTimeBucketer(ZoneId zoneId) {
        this(DEFAULT_FORMAT_STRING, zoneId);
    }
    public PageLogEventTimeBucketer(String formatString, ZoneId zoneId) {
        this.formatString = formatString;
        this.zoneId = zoneId;
        this.dateTimeFormatter = DateTimeFormatter.ofPattern(this.formatString).withZone(this.zoneId);
    }
    //记住，这个方法一定要加，否则dateTimeFormatter对象会是空，此方法会在反序列的时候调用，这样才能正确初始化dateTimeFormatter对象
    //那有的人问了，上面构造函数不是初始化了吗？反序列化的时候是不走构造函数的
    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
        in.defaultReadObject();
        this.dateTimeFormatter = DateTimeFormatter.ofPattern(formatString).withZone(zoneId);
    }
    @Override
    public Path getBucketPath(Clock clock, Path basePath, PageLog element) {
        String newDateTimeString = dateTimeFormatter.format(Instant.ofEpochMilli(element.getuId()*1000));
      // System.out.println(element.getuId());
       // System.out.println(newDateTimeString);
        return new Path(basePath + "/" + newDateTimeString);
    }
}

P_PageLog

package com.flink.app.parJson;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.flink.bean.PageLog;

public class P_PageLog {
    public static PageLog getPageLog(String in){
       // System.out.println(in);
        JSONObject jsonObject = JSON.parseObject(in);
        String app_id = jsonObject.getString("app_id");
        String device_id = jsonObject.getString("device_id");
        String pageId = jsonObject.getString("page_id");
        String uId = jsonObject.getString("uid");
        PageLog pageLog = new PageLog(app_id,device_id,pageId,Long.valueOf(uId));
        return pageLog;
    }
}