主要功能是通过侧数据流 拆分流数据
代码示例
package com.XXXX.driver;
import com.alibaba.flink.connectors.datahub.datastream.source.DatahubSourceFunction;
import com.aliyun.datahub.client.model.RecordEntry;
import com.XXXX.pojo.Aecollectordata;
import com.XXXX.process.GetJsonObject;
import com.XXXX.utils.ConfigPropUtils;
import com.XXXX.utils.getString;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
import java.util.List;
/**
* @author yangyingchun
* @version 1.0
* @date 2023/8/28 16:32
*/
public class SideOutput {
private static String endPoint = "your 's endPoint ";
//private static String endPoint ="public endpoint";//公网访问(填写内网Endpoint,就不用填写公网Endpoint)。
private static String projectName = "projectName ";
private static String topicSourceName = "topicSourceName";
private static String topicSinkName = ConfigPropUtils.get("datahub_sink_topic");
private static String accessId = "accessId ";
private static String accessKey = "accessKey ";
//设置消费的启动位点对应的时间。TimeToStampUtil.timeToStamp("2021-12-21") 此时间至少为当前时间
private static Long datahubStartInMs = System.currentTimeMillis();
// private static Long datahubStartInMs = System.currentTimeMillis();
private static Long datahubEndInMs=Long.MAX_VALUE;
public static void main(String[] args) throws Exception {
//todo 1.获取执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.enableCheckpointing(3600000L);
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 2000L));
env.setParallelism(8);
//todo 2.获取数据
DataStreamSource<List<RecordEntry>> aedata = env.addSource(
new DatahubSourceFunction(
endPoint,
projectName,
topicSourceName,
accessId,
accessKey,
datahubStartInMs,
datahubEndInMs,
20L,
1000L,
1000
));
//todo 3.进行数据转换
DataStream<Aecollectordata> aecollectordataDataStream = aedata.flatMap(new FlatMapFunction<List<RecordEntry>, Aecollectordata>() {
@Override
public void flatMap(List<RecordEntry> value, Collector<Aecollectordata> out) throws Exception {
for (RecordEntry recordEntry : value) {
String timestamp = getString.getString(recordEntry, "timestamp");
String message = getString.getString(recordEntry, "message");
Aecollectordata aecollectordata = GetJsonObject.getAeCollectFastJson(message, timestamp);
out.collect(aecollectordata);
}
}
});
//todo 4.设置侧流数据了标记
//8715891475D0454AAAFED0E901C10178 全员营销
OutputTag<String> allSaleTag = new OutputTag<String>("allSaleTag"){};
//AA5DE272584E436E84207D6FF897B472 O-APP
OutputTag<String> oAppTag = new OutputTag<String>("oAppTag"){};
//6B5D1224809D4BA3B2D397099F870F41 p-APP
OutputTag<String> pAppTag = new OutputTag<String>("pAppTag"){};
//dirty
OutputTag<String> dirtyTag = new OutputTag<String>("dirtyTag"){};
//todo 5.拆分数据流
SingleOutputStreamOperator<String> SideDs = aecollectordataDataStream.process(new ProcessFunction<Aecollectordata, String>() {
@Override
public void processElement(Aecollectordata value, Context ctx, Collector<String> out) throws Exception {
if ("8715891475D0454AAAFED0E901C10178".equals(value.getAppKey())) {
ctx.output(allSaleTag, value.toString());
} else if ("AA5DE272584E436E84207D6FF897B472".equals(value.getAppKey())) {
ctx.output(oAppTag, value.toString());
} else if ("6B5D1224809D4BA3B2D397099F870F41".equals(value.getAppKey())) {
ctx.output(pAppTag, value.toString());
} else if (value.getAppKey() == null) {
ctx.output(dirtyTag, value.toString());
} else {
out.collect(value.toString());
}
}
});
//todo 6.获取侧数据流并打印
DataStream<String> allSaleSide = SideDs.getSideOutput(allSaleTag);
DataStream<String> oraAppSide = SideDs.getSideOutput(oAppTag);
DataStream<String> pickupAppSide = SideDs.getSideOutput(pAppTag);
DataStream<String> dirtySide = SideDs.getSideOutput(dirtyTag);
SideDs.print("主流>>>>>>>>>>>>>");
allSaleSide.print("allSaleSide>>>>>>>>>>>");
oraAppSide.print("oAppTag>>>>>>>>>>>");
pickupAppSide.print("pAppSide>>>>>>>>>>>");
dirtySide.print("dirtySide>>>>>>>>>>>");
//todo 7.启动
env.execute("SideOutput");
}
}