Flink_来自两条流的订单交易匹配(Connect, CoProcessFunction)

        对于订单支付事件，用户支付完成其实并不算完，我们还得确认平台账户上是否到账了。而往往这会来自不同的日志信息，所以我们要同时读入两条流的数据来做合并处理 。这里我们利用connect 将两条流进行连接 , 然后用自定义的CoProcessFunction 进行处理。

import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.co.CoProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

// 定义到账事件样例类
case class ReceiptEvent(txId: String, payChannel: String, timestamp: Long)

object TxMatch {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)

    // 1. 读取订单事件数据
    val orderEventStream = env.readTextFile("D:\\Mywork\\workspace\\Project_idea\\UserBehaviorAnalysis0903\\OrderPayDetect\\src\\main\\resources\\OrderLog.csv")
      .map( data => {
        val arr = data.split(",")
        OrderEvent(arr(0).toLong, arr(1), arr(2), arr(3).toLong)
      } )
      .assignAscendingTimestamps(_.timestamp * 1000L)
      .filter(_.eventType == "pay")
      .keyBy(_.txId)

    // 2. 读取到账事件数据
    val receiptEventStream = env.readTextFile("D:\\Mywork\\workspace\\Project_idea\\UserBehaviorAnalysis0903\\OrderPayDetect\\src\\main\\resources\\ReceiptLog.csv")
      .map( data => {
        val arr = data.split(",")
        ReceiptEvent(arr(0), arr(1), arr(2).toLong)
      } )
      .assignAscendingTimestamps(_.timestamp * 1000L)
      .keyBy(_.txId)

    // 3. 合并两条流，进行处理
    val resultStream = orderEventStream.connect(receiptEventStream)
      .process(new TxPayMatchResult())

    resultStream.print("matched")
    resultStream.getSideOutput(new OutputTag[OrderEvent]("unmatched-pay")).print("unmatched pays")
    resultStream.getSideOutput(new OutputTag[ReceiptEvent]("unmatched-receipt")).print("unmatched receipts")

    env.execute()
  }
}

class TxPayMatchResult() extends CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]{
  // 定义状态，保存当前交易对应的订单支付事件，和到账事件
  lazy val payEventState: ValueState[OrderEvent] = getRuntimeContext.getState(new ValueStateDescriptor[OrderEvent]("pay", classOf[OrderEvent]))
  lazy val receiptEventState: ValueState[ReceiptEvent] = getRuntimeContext.getState(new ValueStateDescriptor[ReceiptEvent]("receipt", classOf[ReceiptEvent]))
  // 侧输出流标签
  val unmatchedPayEventOutputTag = new OutputTag[OrderEvent]("unmatched-pay")
  val unmatchedReceiptEventOutputTag = new OutputTag[ReceiptEvent]("unmatched-receipt")

  override def processElement1(pay: OrderEvent, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]#Context, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
    // 订单支付来了，要判断之前是否有到账事件
    val receipt = receiptEventState.value()
    if (receipt != null) {
      // 如果已经有receipt，正常输出匹配，清空状态
      out.collect((pay, receipt))
      receiptEventState.clear()
      payEventState.clear()
    } else {
      // 如果还没来，注册定时器开始等待5秒
      ctx.timerService().registerEventTimeTimer(pay.timestamp * 1000L + 5000L)
      // 更新状态
      payEventState.update(pay)
    }
  }

  override def processElement2(receipt: ReceiptEvent, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]#Context, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
    // 到账事件来了，要判断之前是否有pay事件
    val pay = payEventState.value()
    if (pay != null) {
      out.collect((pay, receipt))
      receiptEventState.clear()
      payEventState.clear()
    } else {
      // 如果还没来，注册定时器开始等待3秒
      ctx.timerService().registerEventTimeTimer(receipt.timestamp * 1000L + 3000L)
      receiptEventState.update(receipt)
    }
  }

  override def onTimer(timestamp: Long, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]#OnTimerContext, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
    // 定时器触发，判断状态中哪个还存在，就代表另一个没来，输出到侧输出流
    if (payEventState.value() != null){
      ctx.output(unmatchedPayEventOutputTag, payEventState.value())
    }
    if (receiptEventState.value() != null){
      ctx.output(unmatchedReceiptEventOutputTag, receiptEventState.value())
    }
    //清空状态
    receiptEventState.clear()
    payEventState.clear()
  }
}


//输出数据
matched> (OrderEvent(34763,pay,sddf9809ew,1558431068),ReceiptEvent(sddf9809ew,alipay,1558430936))
matched> (OrderEvent(34764,pay,832jksmd9,1558431079),ReceiptEvent(832jksmd9,wechat,1558430938))
matched> (OrderEvent(34765,pay,m23sare32e,1558431082),ReceiptEvent(m23sare32e,wechat,1558430940))
matched> (OrderEvent(34766,pay,92nr903msa,1558431095),ReceiptEvent(92nr903msa,wechat,1558430944))
matched> (OrderEvent(34756,pay,9032n4fd2,1558431951),ReceiptEvent(9032n4fd2,wechat,1558430913))
unmatched receipts> ReceiptEvent(ewr342as4,wechat,1558430845)
unmatched receipts> ReceiptEvent(8fdsfae83,alipay,1558430850)
unmatched pays> OrderEvent(34731,pay,35jue34we,1558430849)
unmatched receipts> ReceiptEvent(sdafen9932,alipay,1558430949)
unmatched pays> OrderEvent(34768,pay,88snrn932,1558430950)