Flink_CEP关于订单超时事件处理

最新推荐文章于 2024-05-06 21:23:44 发布

weixin_43003792

最新推荐文章于 2024-05-06 21:23:44 发布

阅读量787

点赞数

分类专栏： Flink

本文链接：https://blog.csdn.net/weixin_43003792/article/details/115709274

版权

Flink 专栏收录该内容

26 篇文章 0 订阅

订阅专栏

package com

import java.util

import org.apache.flink.cep.{PatternSelectFunction, PatternTimeoutFunction}
import org.apache.flink.cep.scala.{CEP, PatternStream}
import org.apache.flink.cep.scala.pattern.Pattern
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.windowing.time.Time

//订单超时事件处理超时事件报警
object OrderTimeout {
//输入样例类

case class OrderEvent(orderId:Long,enentType:String,txId:String,timestamp:Long)
//输出样例类 orderID 输出消息
case class OrderResult(orderId:Long,resultMsg:String)
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
val dataStream: DataStream[String] = env.readTextFile(“F:\UserBehaviorAnalysis\OrderPayDetect\src\main\resources\OrderLog.csv”)
val orderStream: KeyedStream[OrderEvent, Long] = dataStream.map(data => {
val arr: Array[String] = data.split(",")
OrderEvent(arr(0).toLong, arr(1), arr(2), arr(3).toLong)
}).assignAscendingTimestamps(.timestamp * 1000L)
.keyBy(.orderId)
//主要处理下单完后没有付款的
val orderPattern: Pattern[OrderEvent, OrderEvent] = Pattern
.beginOrderEvent.where(.enentType == “create”)
//非严格紧邻创建订单后可以有其他操作
.followedBy(“pay”).where(.enentType == “pay”)
.within(Time.minutes(5))
val patternStream: PatternStream[OrderEvent] = CEP.pattern(orderStream,orderPattern)
//3.定义侧输出流超时事件
val orderTimeOut = new OutputTagOrderResult
val resultStream: DataStream[OrderResult] = patternStream.select(
orderTimeOut,
new OrderTimeoutSelect(), //用于侧输出流的未匹配成功的事件
new OrderPaySelect() //匹配到的事件
)
resultStream.print(“payed”)
resultStream.getSideOutput(orderTimeOut).print(“timeOut”)
env.execute(“order timeout job”)
}
//未匹配成功的事件放入侧输出流
class OrderTimeoutSelect() extends PatternTimeoutFunction[OrderEvent,OrderResult]{
override def timeout(pattern: util.Map[String, util.List[OrderEvent]], timeoutTimestamp: Long): OrderResult = {
val event: OrderEvent = pattern.get(“create”).iterator().next() //只能提取到create事件不能pay
OrderResult(event.orderId,“timeout”+event.timestamp)
}
}
//匹配成功的输出
class OrderPaySelect() extends PatternSelectFunction[OrderEvent,OrderResult]{
override def select(pattern: util.Map[String, util.List[OrderEvent]]): OrderResult = {
val creatEevent: OrderEvent = pattern.get(“create”).iterator().next()
val orderEvent: OrderEvent = pattern.get(“pay”).iterator().next()
OrderResult(orderEvent.orderId,“下单时间”+creatEevent.timestamp+“支付世间”+orderEvent.timestamp)
}
}
}

package com

import com.OrderTimeout.OrderEvent
import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.co.{CoProcessFunction, KeyedCoProcessFunction}
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

//匹配两条流
object TxMatch {
//支付事件
case class OrderEvent(orderId:Long,enentType:String,txId:String,timestamp:Long)
//到账事件
case class ReceiptEvent(txId:String,payChannel:String,timestamp:Long)

def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
//0.得到数据源
val orderinputStream: DataStream[String] = env.readTextFile(“F:\UserBehaviorAnalysis\OrderPayDetect\src\main\resources\OrderLog.csv”)
//读取订单事件
val orderEvert= orderinputStream.map(data => {
val arr = data.split(",")
OrderEvent(arr(0).toLong, arr(1), arr(2), arr(3).toLong)
}).assignAscendingTimestamps(.timestamp * 1000L).filter(.enentType==“pay”).keyBy(.txId)
//读取到账事件
val receiptStream: DataStream[String] = env.readTextFile(“F:\UserBehaviorAnalysis\OrderPayDetect\src\main\resources\ReceiptLog.csv”)
val receiptEvent= receiptStream.map(data => {
val arr = data.split(",")
ReceiptEvent(arr(0), arr(1), arr(2).toLong)
}).assignAscendingTimestamps(.timestamp * 1000L).keyBy(_.txId)
//合流操作
//connect
//join
val connectionStream: ConnectedStreams[OrderEvent, ReceiptEvent] = orderEvert.connect(receiptEvent)
val resultStream: DataStream[(OrderEvent, ReceiptEvent)] = connectionStream.process(new TxPayMatchResult)
resultStream.print(“match”)
resultStream.getSideOutput(new OutputTagOrderEvent).print(“unpay”)
resultStream.getSideOutput(new OutputTagReceiptEvent).print(“onmathc”)
env.execute(“TxMatch”)

}
class TxPayMatchResult extends CoProcessFunction[OrderEvent,ReceiptEvent,(OrderEvent,ReceiptEvent)]{
// 定义状态，保存当前交易对应的订单支付事件和到账事件
lazy val payEventState: ValueState[OrderEvent] = getRuntimeContext.getState(new ValueStateDescriptor[OrderEvent](“pay”, classOf[OrderEvent]))
lazy val receiptEventState: ValueState[ReceiptEvent] = getRuntimeContext.getState(new ValueStateDescriptor[ReceiptEvent](“receipt”, classOf[ReceiptEvent]))
// 侧输出流标签
val unmatchedPayEventOutputTag = new OutputTagOrderEvent
val unmatchedReceiptEventOutputTag = new OutputTagReceiptEvent

override def processElement1(pay: OrderEvent, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]#Context, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
  // 订单支付来了，要判断之前是否有到账事件
  val receipt = receiptEventState.value()
  if( receipt != null ){
    // 如果已经有receipt，正常输出匹配，清空状态
    out.collect((pay, receipt))
    receiptEventState.clear()
    payEventState.clear()
  } else{
    // 如果还没来，注册定时器开始等待5秒
    ctx.timerService().registerEventTimeTimer(pay.timestamp * 1000L + 5000L)
    // 更新状态
    payEventState.update(pay)
  }
}

override def processElement2(receipt: ReceiptEvent, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]#Context, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
  // 到账事件来了，要判断之前是否有pay事件
  val pay = payEventState.value()
  if( pay != null ){
    // 如果已经有pay，正常输出匹配，清空状态
    out.collect((pay, receipt))
    receiptEventState.clear()
    payEventState.clear()
  } else{
    // 如果还没来，注册定时器开始等待3秒
    ctx.timerService().registerEventTimeTimer(receipt.timestamp * 1000L + 3000L)
    // 更新状态
    receiptEventState.update(receipt)
  }
}

override def onTimer(timestamp: Long, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]#OnTimerContext, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
  // 定时器触发，判断状态中哪个还存在，就代表另一个没来，输出到侧输出流
  if( payEventState.value() != null ){
    ctx.output(unmatchedPayEventOutputTag, payEventState.value())
  }
  if( receiptEventState.value() != null ){
    ctx.output(unmatchedReceiptEventOutputTag, receiptEventState.value())
  }
  // 清空状态
  receiptEventState.clear()
  payEventState.clear()

}
}
}

package com

import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.co.{CoProcessFunction, ProcessJoinFunction}
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.util.Collector

//匹配两条流
object TxMatchWithJoin {
//支付事件
case class OrderEvent(orderId:Long,enentType:String,txId:String,timestamp:Long)
//到账事件
case class ReceiptEvent(txId:String,payChannel:String,timestamp:Long)

def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
//0.得到数据源
val orderinputStream: DataStream[String] = env.readTextFile(“F:\UserBehaviorAnalysis\OrderPayDetect\src\main\resources\OrderLog.csv”)
//读取订单事件
val orderEvert= orderinputStream.map(data => {
val arr = data.split(",")
OrderEvent(arr(0).toLong, arr(1), arr(2), arr(3).toLong)
}).assignAscendingTimestamps(.timestamp * 1000L).filter(.enentType==“pay”).keyBy(.txId)
//读取到账事件
val receiptStream: DataStream[String] = env.readTextFile(“F:\UserBehaviorAnalysis\OrderPayDetect\src\main\resources\ReceiptLog.csv”)
val receiptEvent= receiptStream.map(data => {
val arr = data.split(",")
ReceiptEvent(arr(0), arr(1), arr(2).toLong)
}).assignAscendingTimestamps(.timestamp * 1000L).keyBy(_.txId)
//合流操作
//connect
//join
val result: DataStream[(OrderEvent, ReceiptEvent)] = orderEvert.intervalJoin(receiptEvent)
.between(Time.seconds(-3), Time.seconds(5))
.process(new TxMatchWithJoinResult())
result.print(“匹配成功”)
env.execute(“TxMatch”)
}
class TxMatchWithJoinResult() extends ProcessJoinFunction[OrderEvent,ReceiptEvent,(OrderEvent,ReceiptEvent)]{
override def processElement(left: OrderEvent, right: ReceiptEvent, ctx: ProcessJoinFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]#Context, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
out.collect(left,right)
}

}
}

package com

import java.util

import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.cep.scala.pattern.Pattern
import org.apache.flink.cep.scala.{CEP, PatternStream}
import org.apache.flink.cep.{PatternSelectFunction, PatternTimeoutFunction}
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.util.Collector

//订单超时事件处理
object OrderTimeoutWithState {
case class OrderEvent(orderId:Long,enentType:String,txId:String,timestamp:Long)
case class OrderResult(orderId:Long,resultMsg:String)
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
//0.得到数据yuan
val inputStream: DataStream[String] = env.readTextFile(“F:\UserBehaviorAnalysis\OrderPayDetect\src\main\resources\OrderLog.csv”)
val orderEventStream: KeyedStream[OrderEvent, Long] = inputStream.map(data => {
val arr = data.split(",")
OrderEvent(arr(0).toLong, arr(1), arr(2), arr(3).toLong)
}).assignAscendingTimestamps(.timestamp * 1000L)
.keyBy(.orderId)
val result: DataStream[OrderResult] = orderEventStream.process(new OrderPayMatchResult())
result.print(“payed”)

result.getSideOutput(new OutputTag[OrderResult]("timeout")).print("timeout")
env.execute()

}
class OrderPayMatchResult() extends KeyedProcessFunction[Long,OrderEvent,OrderResult]{
//定义状态

//1.如果pay来了 判断create来没
lazy val isCreateState: ValueState[Boolean] = getRuntimeContext.getState(new ValueStateDescriptor[Boolean]("is-create",classOf[Boolean]))
lazy val isPayedState: ValueState[Boolean] = getRuntimeContext.getState(new ValueStateDescriptor[Boolean]("is-payed",classOf[Boolean]))
lazy val timerTsState: ValueState[Long] = getRuntimeContext.getState(new ValueStateDescriptor[Long]("ts",classOf[Long]))
//
lazy val orderTimeoutOutputTag = new OutputTag[OrderResult]("timeout")
override def processElement(value: OrderEvent, ctx: KeyedProcessFunction[Long, OrderEvent, OrderResult]#Context, out: Collector[OrderResult]): Unit = {
  val isCreate: Boolean = isCreateState.value()
  val isPay: Boolean = isPayedState.value()
  val timeTs: Long = timerTsState.value()
  //判断当前事件 是create 还是pay
  if(value.enentType == "create"){
    //1.1如果已经支付过 正常支付 输出匹配成功结果 清空所有状态和定时器
    if(isPay){out.collect(OrderResult(value.orderId,"pay-sussess"))
      isCreateState.clear()
      isPayedState.clear()
      ctx.timerService().deleteEventTimeTimer(timeTs)
      }else{
      //1.2如果每来过 注册定时器 等待15分钟
      val ts: Long = value.timestamp*1000L+900*1000L
      ctx.timerService().registerEventTimeTimer(ts)
    }

  }
    //2.如果是pay
  else if(value.enentType == "pay"){
      //判断是否来过create
    if(isCreate){
      //2.1如果create过 匹配成功 判断pay的时间和create的时间
      if(value.timestamp*1000L<timeTs){out.collect(OrderResult(value.orderId,"payed sucess"))}
      //2.2 如果超时
      else{ctx.output(orderTimeoutOutputTag,OrderResult(value.orderId,"pay but outtime"))}
      //只要输出结果 当前order
      isCreateState.clear()
      isPayedState.clear()
      ctx.timerService().deleteEventTimeTimer(timeTs)
    }
  }
    //2.2如果create没 来
  else {
    ctx.timerService().registerEventTimeTimer(value.timestamp*1000L)
    //更新状态
    timerTsState.update(value.timestamp*1000L)
    isPayedState.update(true)
  }
}

override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[Long, OrderEvent, OrderResult]#OnTimerContext, out: Collector[OrderResult]): Unit = {
  //定时器触发
  //1.pay来了 没等到create
  if(isPayedState.value()){
    ctx.output(orderTimeoutOutputTag,OrderResult(ctx.getCurrentKey,"pay but no create"))
  }else{
    ctx.output(orderTimeoutOutputTag,OrderResult(ctx.getCurrentKey,"timeout"))
  }
  //清空所有状态
  isCreateState.clear()
  isPayedState.clear()
  timerTsState.clear()
}

}
}