案例1:对传播信号进行分流,速度大于15的走主流,小于15的走测输出流
object splitAIS {
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
val KafkaConsumer: KafkaSource[String] = KafkaSource.builder[String]()
.setBootstrapServers("192.168.126.51:9092,192.168.126.52:9092,192.168.126.53:9092")
.setGroupId("zy")
.setTopics("flink")
.setStartingOffsets(OffsetsInitializer.latest())
.setValueOnlyDeserializer(new SimpleStringSchema())
.build()
val value: DataStream[String] = env.fromSource(KafkaConsumer, WatermarkStrategy.noWatermarks(), "kafka")
// value.print()
val value1: DataStream[aisCase] = value
.filter(!_.contains("mmsi"))
.map(x => {
val strings: Array[String] = x.split(",")
aisCase(strings(0), strings(1), strings(2), strings(3),
strings(4).toDouble, strings(5), strings(6))
})
.process(new aisSplit)
//分流操作不对数据做任何筛选操作
//打印主流
value1.print("高速")
//打印测输出流
value1.getSideOutput(new OutputTag[aisCase]("低速")).print("低速:")
env.execute()
}
}
class aisSplit extends ProcessFunction[aisCase,aisCase]{
override def processElement(i: aisCase,
context: ProcessFunction[aisCase, aisCase]#Context,
collector: Collector[aisCase]): Unit = {
if (i.speed<15){
//走测输出流
context.output(new OutputTag[aisCase]("低速"),i)
}else {
//走主流
collector.collect(i)
}
}
}
案例2:对日志进行分流,响应时间>1走测输出流
object splitLog {
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
val KafkaConsumer: KafkaSource[String] = KafkaSource.builder[String]()
.setBootstrapServers("192.168.126.51:9092,192.168.126.52:9092,192.168.126.53:9092")
.setGroupId("zy")
.setTopics("flink")
.setStartingOffsets(OffsetsInitializer.latest())
.setValueOnlyDeserializer(new SimpleStringSchema())
.build()
val value: DataStream[String] = env.fromSource(KafkaConsumer, WatermarkStrategy.noWatermarks(), "kafka")
val value1: DataStream[(String, Double)] = value
.map(x => {
val strings: Array[String] = x.split(" ") //\t 是Tab大空格
(strings(9), strings(6).toDouble)
})
.keyBy(_._1)
.process(new logSplit)
value1
//打印主流
.print("快响应:")
env.execute()
}
}
class logSplit extends KeyedProcessFunction[String,(String,Double),(String,Double)]{
override def processElement(i: (String, Double),
context: KeyedProcessFunction[String, (String, Double), (String, Double)]#Context,
collector: Collector[(String, Double)]): Unit ={
if (i._2>1){
//走测输出流
context.output(new OutputTag[(String,Double)]("慢响应"),i)
}else{
//走主流
collector.collect(i)
}
}
}