flnk hive

import com.alibaba.fastjson.JSON;
import com.hihonor.cloud.crypto.utils.CommonClass;
import com.hihonor.cloud.crypto.utils.ConfigUtils;
import com.hihonor.cloud.crypto.utils.StreamExecutionEnvironmentIUtils;
import com.hihonor.cloud.three.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
import org.apache.log4j.Logger;

import java.util.*;

public class SinkToThreeTables {
private static final Logger LOG = Logger.getLogger(SinkToThreeTables.class);
public static void main(String[] args) {
StreamExecutionEnvironment env = StreamExecutionEnvironmentIUtils.getConfigEnv();
env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
env.enableCheckpointing(600000, CheckpointingMode.EXACTLY_ONCE);
env.setRuntimeMode(RuntimeExecutionMode.STREAMING);
env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
String path = ConfigUtils.getPath();
if (StringUtils.isEmpty(path)) {
LOG.info(“obs checkpoint path must be configured:” + path);
}
env.setStateBackend(new FsStateBackend(path));
String kafkaTopic = LoadKafkaConfig.getKafkaTopic();
Properties kafkaProps = LoadKafkaConfig.loadCommonConfig();
if (StringUtils.isEmpty(kafkaTopic)||kafkaProps.isEmpty()){
LOG.info(“kafka topic is missing:”);
}
Properties tag1Props = TagJsonUtils.getTagProps(“tag1”);
Properties tag2Props = TagJsonUtils.getTagProps(“tag2”);
Properties tag3Props = TagJsonUtils.getTagProps(“tag3”);
Properties tag1Prop = TagDataUtils.getTagProps(“tag1”);
Properties tag2Prop = TagDataUtils.getTagProps(“tag2”);
Properties tag3Prop = TagDataUtils.getTagProps(“tag3”);
if (tag1Propsnull||tag1Props.isEmpty()){
System.out.println("=tag1Props="+ JSON.toJSONString(tag1Props));
return ;
}
if (tag2Propsnull||tag2Props.isEmpty()){
System.out.println("=tag2Props="+ JSON.toJSONString(tag2Props));
return ;
}
if (tag3Propsnull||tag3Props.isEmpty()){
System.out.println("=tag3Props="+ JSON.toJSONString(tag3Props));
return ;
}
if (tag1Propnull||tag1Prop.isEmpty()){
System.out.println("=tag1Prop="+ JSON.toJSONString(tag1Prop));
return ;
}
if (tag2Propnull||tag2Prop.isEmpty()){
System.out.println("=tag2Prop="+ JSON.toJSONString(tag2Prop));
return ;
}
if (tag3Propnull||tag3Prop.isEmpty()){
System.out.println("=tag3Prop="+ JSON.toJSONString(tag3Prop));
return ;
}
FlinkKafkaConsumerBase kafkaConsumerBase = new FlinkKafkaConsumer(
kafkaTopic,
new SimpleStringSchema(),
kafkaProps)
.setStartFromLatest();
kafkaConsumerBase.setCommitOffsetsOnCheckpoints(true);
SingleOutputStreamOperator kafkaStream = env.addSource(
kafkaConsumerBase
);
OutputTag tag1 = new OutputTag(“tag1”, TypeInformation.of(CommonClass.class)){};
OutputTag tag2 = new OutputTag(“tag2”, TypeInformation.of(CommonClass.class)){};
OutputTag tag3 = new OutputTag(“tag3”, TypeInformation.of(CommonClass.class)){};
//确定这个地方出现了问题,数据是没有打印出来的。但是下面确实是产生了数据的
//测试了下面的数据都是可以正常数据的
SingleOutputStreamOperator process = kafkaStream.flatMap(new FlatMapFunction<String, Map<String, List>>() {
@Override
public void flatMap(String value, Collector<Map<String, List>> out) throws Exception {
Map<String, List> map = new HashMap<>();
List tagData1 = getTag1Data(value, tag1Props, tag1Prop);
System.out.println(“tagData1” + JSON.toJSONString(tagData1));
if (tagData1 != null && tagData1.size() > 0) {
map.put(“tag1”, tagData1);
}
List tagData2 = getTag2Data(value, tag2Props, tag2Prop);
System.out.println(“tagData2” + JSON.toJSONString(tagData2));
if (tagData2 != null && tagData2.size() > 0) {
map.put(“tag2”, tagData2);
}
List tagData3 = getTag3Data(value, tag3Props, tag3Prop);
System.out.println(“tagData3” + JSON.toJSONString(tagData3));
if (tagData3 != null && tagData3.size() > 0) {
map.put(“tag3”, tagData3);
}
out.collect(map);
}
}).process(new ProcessFunction<Map<String, List>, CommonClass>() {
@Override
public void processElement(Map<String, List> value, Context ctx, Collector out) throws Exception {
Set<Map.Entry<String, List>> entries = value.entrySet();
Iterator<Map.Entry<String, List>> iterator = entries.iterator();
while (iterator.hasNext()) {
Map.Entry<String, List> next = iterator.next();
String key = next.getKey();
if (StringUtils.equals(“tag1”, key)) {
List tag1Value = next.getValue();
System.out.println(“tag1” + JSON.toJSONString(tag1Value));
for (CommonClass commonClass : tag1Value) {
ctx.output(tag1, commonClass);
}
}
if (StringUtils.equals(“tag2”, key)) {
List tag2Value = next.getValue();
System.out.println(“tag2” + JSON.toJSONString(tag2Value));
for (CommonClass commonClass : tag2Value) {
ctx.output(tag2, commonClass);
}
}
if (StringUtils.equals(“tag3”, key)) {
List tag3Value = next.getValue();
System.out.println(“tag3” + JSON.toJSONString(tag3Value));
for (CommonClass commonClass : tag3Value) {
ctx.output(tag3, commonClass);
}
}
}
}
});
//这一段是无法执行的操作的.需要进一步的排查数据问题
//数据格式还是存在很多的问题的,需要进一步的排查数据问题
DataStream tag1Stream = process.getSideOutput(tag1);
DataStream tag2Stream = process.getSideOutput(tag2);
DataStream tag3Stream = process.getSideOutput(tag3);
tag1SinkToHive(tag1Stream,tag1Props,tag1Prop);
tag2SinkToHive(tag2Stream,tag2Props,tag2Prop);
tag3SinkToHive(tag3Stream,tag3Props,tag3Prop);
try {
env.execute(“sink to hive tables”);
} catch (Exception e) {
LOG.info(“job exec failed”+e.getMessage());
}
}

private static List<CommonClass> getTag3Data(String value, Properties tag3Props, Properties tag3Prop) {
    return CollectDataUtils.getDataForTag3(value,tag3Props,tag3Prop);
}

private static List<CommonClass> getTag2Data(String value, Properties tag2Props, Properties tag2Prop) {
    return CollectDataUtils.getDataForTag2(value,tag2Props,tag2Prop);
}

private static List<CommonClass> getTag1Data(String value, Properties tag1Props, Properties tag1Prop) {
    return CollectDataUtils.getDataForTag1(value,tag1Props,tag1Prop);
}

//数据入湖操作实现
private static void tag3SinkToHive(DataStream<CommonClass> tag3Stream, Properties tag3Props, Properties tag3Prop) {
    SingleOutputStreamOperator<Tag3Common> tag3Data = tag3Stream.flatMap(new RichFlatMapFunction<CommonClass, Tag3Common>() {
        @Override
        public void flatMap(CommonClass value, Collector<Tag3Common> out) throws Exception {
            Tag3Common tag3Common = DataTransformUtils.transformDataForTag3(value);
            out.collect(tag3Common);
        }
    }).filter(x->x!=null);
    sinkToHiveForTag3(tag3Data,tag3Props,"tag3",tag3Prop);
}

private static void sinkToHiveForTag3(SingleOutputStreamOperator<Tag3Common> tag3Data, Properties tag3, String s, Properties tag3Prop) {
    HiveUtils.sinkForTag3(tag3Data,tag3,s,tag3Prop);
}

//数据入湖操作实现
private static void tag2SinkToHive(DataStream<CommonClass> tag2Stream, Properties tag2Props, Properties tag2Prop) {
    SingleOutputStreamOperator<Tag2Common> tag3Data = tag2Stream.flatMap(new RichFlatMapFunction<CommonClass, Tag2Common>() {
        @Override
        public void flatMap(CommonClass value, Collector<Tag2Common> out) throws Exception {
            Tag2Common tag2Common = DataTransformUtils.transformDataForTag2(value);
            out.collect(tag2Common);
        }
    }).filter(x->x!=null);
    sinkToHiveForTag2(tag3Data,tag2Props,"tag2",tag2Prop);
}

private static void sinkToHiveForTag2(SingleOutputStreamOperator<Tag2Common> tag2Data, Properties tag2, String s, Properties tag2Prop) {
    HiveUtils.sinkForTag2(tag2Data,tag2,s,tag2Prop);
}

//数据入湖操作实现
private static void tag1SinkToHive(DataStream<CommonClass> tag1Stream, Properties tag1Prop, Properties prop) {
    SingleOutputStreamOperator<Tag1Common> tag1Data = tag1Stream.flatMap(new RichFlatMapFunction<CommonClass, Tag1Common>() {
        @Override
        public void flatMap(CommonClass value, Collector<Tag1Common> out) throws Exception {
            Tag1Common tag1Common = DataTransformUtils.transformDataForTag1(value);
            out.collect(tag1Common);
        }
    }).filter(x->x!=null);
    sinkToHiveForTag1(tag1Data,tag1Prop,"tag1",prop);
}

private static void sinkToHiveForTag1(SingleOutputStreamOperator<Tag1Common> tag1Data, Properties tag1, String s, Properties prop) {
    HiveUtils.sinkForTag1(tag1Data,tag1,s,prop);
}

}