此处简单起见,使用processTime来处理。
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
public class join {
public join() {
}
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
DataStream<String> dataStream1 = env.socketTextStream("10.203.0.53", 8000);
DataStream<String> dataStream2 = env.socketTextStream("10.203.0.53", 8080);
//获取dataStream1
DataStream<Tuple2<String,String>> ds1 = dataStream1.filter(new FilterFunction<String>() {
@Override
public boolean filter(String s) throws Exception {
if(s != "" || s.split(",").length == 2 || s != null){
return true;
} else {
return false;
}
}
}).map(new MapFunction<String, Tuple2<String,String>>() {
@Override
public Tuple2<String, String> map(String s) throws Exception {
return new Tuple2<String,String>(s.split(",")[0],s.split(",")[1]);
}
});
//获取dataStream2
DataStream<Tuple2<String,String>> ds2 = dataStream2.filter(new FilterFunction<String>() {
@Override
public boolean filter(String s) throws Exception {
if(s != null || s.split(",").length == 2){
return true;
}
return false;
}
}).map(new MapFunction<String, Tuple2<String,String>>() {
@Override
public Tuple2<String, String> map(String s) throws Exception {
return new Tuple2<String,String>(s.split(",")[0],s.split(",")[1]);
}
});
//开始根据PorcessTime时间窗口进行JOIN,5秒进行一次
//where和 equalTo实际是2个stream的关联字段,此处为Tuple2.f0
ds1.join(ds2).where(new KeySelector<Tuple2<String, String>, String>() {
@Override
public String getKey(Tuple2<String, String> stringStringTuple2) throws Exception {
return stringStringTuple2.f0;
}
}).equalTo(new KeySelector<Tuple2<String, String>, String>() {
@Override
public String getKey(Tuple2<String, String> stringStringTuple2) throws Exception {
return stringStringTuple2.f0;
}
}).window(TumblingProcessingTimeWindows.of(Time.seconds(5)))
.apply(new JoinFunction<Tuple2<String, String>, Tuple2<String, String>, String>() {
@Override
public String join(Tuple2<String, String> stringStringTuple2, Tuple2<String, String> stringStringTuple22) throws Exception {
return stringStringTuple2.f1 + "," + stringStringTuple22.f1;
}
}
).print();
env.execute("Window join");
}
}
这里例子是开启了2个socket,根据输入的字符,通过逗号分隔成Tuple2,然后根据 Tuple2.f0进行inner join,最后输入Tuple2.f1 的值。
window join有一个问题需要思考,因为所有数据在同一个窗口进行处理,假设多个数据流JOIN,某个数据流数据延迟,那么数据就不在同一个窗口,会导致在JOIN的时候漏掉了,这是个非常严重的问题,和数据丢失了一样。
比如Stream A和 Stream B进行JOIN,A表是订单,B表示物流信息,在业务系统当中,我们下单的数据肯定是进入ORDER数据库,物流信息肯定是进入物流的数据库,如果要查询某个订单发货到哪里,这种信息需要查询ORDER和物流数据库才可以得到,在数据库当中这种信息不会延迟很久,属于毫秒级,最多 也就1秒左右就能查询出来。
但是在流处理的时候确不能保证,因为数据发送到KAFKA,我们无法确保在这2者数据在同一个窗口,一旦出现这个情况,window join就会出现数据查询丢失问题。
但是单流的方式配合维度表的场景不多,一般都是多流方式 ,所以这个问题目前还不清楚如何处理。一旦数据延迟,数据不在同一个窗口,该如何处理。