Flink之用户行为分析之两条流的订单交易匹配

---

订单交易匹配

对于订单支付事件，用户支付完成其实并不算完，我们还得确认平台账户上是否到账了。而往往这会来自不同的日志信息，所以我们要同时读入两条流的数据来做合并 处 理 。 这 里 我 们 利 用 connect 将两 条 流 进 行 连 接 ， 然 后 用 自 定 义 的CoProcessFunction 进行处理。

case class OrderEvent( orderId: Long, eventType: String, txId: String, eventTime: Long )
case class ReceiptEvent( txId: String, payChannel: String, eventTime: Long )
object TxMatch {
 val unmatchedPays = new OutputTag[OrderEvent]("unmatchedPays")
 val unmatchedReceipts = new OutputTag[ReceiptEvent]("unmatchedReceipts")
 def main(args: Array[String]): Unit = {
 val env = StreamExecutionEnvironment.getExecutionEnvironment
 env.setParallelism(1)
 env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
 val orderEventStream = env.readTextFile("YOUR_PATH\\resources\\OrderLog.csv")
.map( data => {
val dataArray = data.split(",")
OrderEvent(dataArray(0).toLong, dataArray(1), dataArray(2),
dataArray(3).toLong)
})
.filter(_.txId != "")
 .assignAscendingTimestamps(_.eventTime * 1000L)
 .keyBy(_.txId)
 val receiptEventStream = env.readTextFile("YOUR_PATH\\resources\\ReceiptLog.csv")
.map( data => {
val dataArray = data.split(",")
ReceiptEvent(dataArray(0), dataArray(1), dataArray(2).toLong)
})
 .assignAscendingTimestamps(_.eventTime * 1000L)
 .keyBy(_.txId)
 val processedStream = orderEventStream
 .connect(receiptEventStream)
 .process(new TxMatchDetection)
 processedStream.getSideOutput(unmatchedPays).print("unmatched pays")
 processedStream.getSideOutput(unmatchedReceipts).print("unmatched receipts")
 processedStream.print("processed")
 env.execute()
 }
 class TxMatchDetection extends CoProcessFunction[OrderEvent, ReceiptEvent, 
(OrderEvent, ReceiptEvent)]{
 lazy val payState: ValueState[OrderEvent] = getRuntimeContext.getState(new 
ValueStateDescriptor[OrderEvent]("pay-state",classOf[OrderEvent]) )
lazy val receiptState: ValueState[ReceiptEvent] = getRuntimeContext.getState(new 
ValueStateDescriptor[ReceiptEvent]("receipt-state", classOf[ReceiptEvent]) )
 override def processElement1(pay: OrderEvent, ctx: CoProcessFunction[OrderEvent, 
ReceiptEvent, (OrderEvent, ReceiptEvent)]#Context, out: Collector[(OrderEvent, 
ReceiptEvent)]): Unit = {
 val receipt = receiptState.value()
 if( receipt != null ){
 receiptState.clear()
 out.collect((pay, receipt))
 } else{
 payState.update(pay)
 ctx.timerService().registerEventTimeTimer(pay.eventTime * 1000L)
 }
 }
 override def processElement2(receipt: ReceiptEvent, ctx: 
CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]#Context, out: 
Collector[(OrderEvent, ReceiptEvent)]): Unit = {
 val payment = payState.value()
 if( payment != null ){
 payState.clear()
 out.collect((payment, receipt))
 } else{
 receiptState.update(receipt)
 ctx.timerService().registerEventTimeTimer(receipt.eventTime * 1000L)
 }
}
 override def onTimer(timestamp: Long, ctx: CoProcessFunction[OrderEvent, 
ReceiptEvent, (OrderEvent, ReceiptEvent)]#OnTimerContext, out: Collector[(OrderEvent, 
ReceiptEvent)]): Unit = {
 if ( payState.value() != null ){
 ctx.output(unmatchedPays, payState.value())
 }
 if ( receiptState.value() != null ){
 ctx.output(unmatchedReceipts, receiptState.value())
 }
 payState.clear()
 receiptState.clear()
 }
 }
}

总结

代码理解：两条流的输入，一条是orderEvent，一条是接收流ReceiptEvent。对于订单事件流，先做filter操作，过滤出不为空的事件，再根据txId进行keyBy。对于支付到账事件流，先按照时间流升序排序，再同样根据Id进行KeyBy，如果两个id对的上，那么就是成功到账信息，则可以进行输出。如果一个没有，另一个有，则输出报警信息，用cenect将两条流数据连接起来共同处理。