package cn.edu.tju.demo;
import org.apache.flink.api.common.functions.*;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.scala.codegen.TypeDescriptors;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.runtime.state.changelog.StateChange;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.checkpoint.ListCheckpointed;
import org.apache.flink.streaming.api.collector.selector.OutputSelector;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.streaming.api.functions.co.CoMapFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.redis.RedisSink;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
import org.apache.http.HttpHost;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;
import java.util.*;
public class Test29 {
private static String HOST_NAME = "xx.xx.xx.xx";
private static int PORT = 9999;
private static String DELIMITER ="\n";
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment environment = StreamExecutionEnvironment
.getExecutionEnvironment();
//environment.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
DataStream<String> socketDataInfo = environment.socketTextStream(HOST_NAME, PORT, DELIMITER);
SingleOutputStreamOperator<DataInfo> dataInfoStream = socketDataInfo.map(new MapFunction<String, DataInfo>() {
@Override
public DataInfo map(String value) throws Exception {
String[] stringList = value.split(",");
DataInfo dataInfo = new DataInfo(Long.parseLong(
stringList[0]), stringList[1], Double.parseDouble(stringList[2]));
return dataInfo;
}
});
OutputTag<DataInfo> myOutputTag = new OutputTag<DataInfo>("low"){};
SingleOutputStreamOperator<DataInfo> resultStream = dataInfoStream.process(new ProcessFunction<DataInfo, DataInfo>() {
@Override
public void processElement(DataInfo value, ProcessFunction<DataInfo, DataInfo>.Context ctx, Collector<DataInfo> out) throws Exception {
//value大于10,分流到主流resultStream,否则分到侧流low
if (value.getVal() > 10) {
out.collect(value);
} else {
ctx.output(myOutputTag, value);
}
}
});
resultStream.print("result");
resultStream.getSideOutput(myOutputTag).print("side");
environment.execute("my job");
}
public static class DataInfo{
private long ts;
private String info;
private double val;
public long getTs() {
return ts;
}
public void setTs(long ts) {
this.ts = ts;
}
public String getInfo() {
return info;
}
public void setInfo(String info) {
this.info = info;
}
public double getVal() {
return val;
}
public void setVal(double val) {
this.val = val;
}
@Override
public String toString() {
return "DataInfo{" +
"ts=" + ts +
", info='" + info + '\'' +
", val='" + val + '\'' +
'}';
}
public DataInfo(long ts, String info, double val) {
this.ts = ts;
this.info = info;
this.val = val;
}
public DataInfo() {
}
}
}
flink: 使用ProcessFunction实现数据分流(sideOutput)
最新推荐文章于 2024-10-09 23:33:19 发布