基于 DataStream API 实现欺诈检测、Flink的定时器

基于 DataStream API 实现欺诈检测

实时场景

Flink的定时器

package com.shujia.flink.core

import org.apache.flink.api.common.functions.RuntimeContext
import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.TimerService
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

object Demo6Fraud {
  def main(args: Array[String]): Unit = {

    /*
    数据：
    用户编号，操作金额
    1001,123
    1001,23
    1001,1
    1001,1000
    1001,300
    1002,2
    1002,0.5
    1002,200
    1002,0.5
    1002,6000
    */

    /**
      * 对于一个账户，如果出现小于 $1 美元的交易后紧跟着一个大于 $500 的交易，就输出一个报警信息。
      *
      */
    
    //创建Flink运行环境
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    //读取数据 -- 开启socket
    val eventDS: DataStream[String] = env.socketTextStream("master", 8888)
    //处理数据   
    val kvDS: DataStream[(String, Double)] = eventDS.map(line => {
      val split: Array[String] = line.split(",")
      (split(0), split(1).toDouble)
    })

    //按照用户分组
    val keyByDS: KeyedStream[(String, Double), String] = kvDS.keyBy(_._1)

    /**
      * 进行欺诈检查
      * 返回结果
      * 用户编号，前一次的金额，后一次的金额
      *
      * 如果出现小于 $1 美元的交易后紧跟着一个大于 $500 的交易，就输出一个报警信息。
      *
      * 两次行为必须在一分钟内完成才发出报警
      *
      */

    val filterDS: DataStream[(String, Double, Double)] = keyByDS.process(new KeyedProcessFunction[String, (String, Double), (String, Double, Double)] {

      /**
        * 用来保存金额小于1的数据
        *
        */
      var valueState: ValueState[Double] = _

      override def open(parameters: Configuration): Unit = {

        val context: RuntimeContext = getRuntimeContext

        valueState = context.getState(new ValueStateDescriptor[Double]("money", classOf[Double]))
      }

      override def processElement(
                                   value: (String, Double),
                                   ctx: KeyedProcessFunction[String, (String, Double), (String, Double, Double)]#Context,
                                   out: Collector[(String, Double, Double)]): Unit = {

        val (id, money) = value

        //获取上一次的金额
        val lastMoney: Double = valueState.value()

        if (lastMoney != 0) {
          if (money > 500) {
            //出一个报警信息。
            out.collect((id, lastMoney, money))
          } else {
            //恢复到初始状态
            valueState.update(0)
          }
        }

        if (money < 1.0) {
          //将当前的金额保存到状态中
          valueState.update(money)


          println("注册定时器")
          //获取定时器对象
          val timerService: TimerService = ctx.timerService()
          //获取当前处理事件
          val currTime: Long = timerService.currentProcessingTime()

          //注册定时器 -- 可以使用会话窗口
          /**
            *
            * 同时只能有一个定时器，按照最新的为准 -- 所以不删除定时器也行
            *
            * 当时间达到指定时间之后会触发onTimer方法的执行
            *
            */
          //一分钟之后触发onTimer执行
          timerService.registerProcessingTimeTimer(currTime + 60000)

        }
      }

      override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, (String, Double), (String, Double, Double)]#OnTimerContext, out: Collector[(String, Double, Double)]): Unit = {
        println("onTimer")
        //恢复到初始状态
        valueState.update(0)
      }
    })

    filterDS.print()

    env.execute()

  }
}

下面附上官网的完整程序

package spendreport

import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.api.scala.typeutils.Types
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.util.Collector
import org.apache.flink.walkthrough.common.entity.Alert
import org.apache.flink.walkthrough.common.entity.Transaction

object FraudDetector {
  val SMALL_AMOUNT: Double = 1.00
  val LARGE_AMOUNT: Double = 500.00
  val ONE_MINUTE: Long     = 60 * 1000L
}

@SerialVersionUID(1L)
class FraudDetector extends KeyedProcessFunction[Long, Transaction, Alert] {

  @transient private var flagState: ValueState[java.lang.Boolean] = _
  @transient private var timerState: ValueState[java.lang.Long] = _

  @throws[Exception]
  override def open(parameters: Configuration): Unit = {
    val flagDescriptor = new ValueStateDescriptor("flag", Types.BOOLEAN)
    flagState = getRuntimeContext.getState(flagDescriptor)

    val timerDescriptor = new ValueStateDescriptor("timer-state", Types.LONG)
    timerState = getRuntimeContext.getState(timerDescriptor)
  }

  override def processElement(
      transaction: Transaction,
      context: KeyedProcessFunction[Long, Transaction, Alert]#Context,
      collector: Collector[Alert]): Unit = {

    // Get the current state for the current key
    val lastTransactionWasSmall = flagState.value

    // Check if the flag is set
    if (lastTransactionWasSmall != null) {
      if (transaction.getAmount > FraudDetector.LARGE_AMOUNT) {
        // Output an alert downstream
        val alert = new Alert
        alert.setId(transaction.getAccountId)

        collector.collect(alert)
      }
      // Clean up our state
      cleanUp(context)
    }

    if (transaction.getAmount < FraudDetector.SMALL_AMOUNT) {
      // set the flag to true
      flagState.update(true)
      val timer = context.timerService.currentProcessingTime + FraudDetector.ONE_MINUTE

      context.timerService.registerProcessingTimeTimer(timer)
      timerState.update(timer)
    }
  }

  override def onTimer(
      timestamp: Long,
      ctx: KeyedProcessFunction[Long, Transaction, Alert]#OnTimerContext,
      out: Collector[Alert]): Unit = {
    // remove flag after 1 minute
    timerState.clear()
    flagState.clear()
  }

  @throws[Exception]
  private def cleanUp(ctx: KeyedProcessFunction[Long, Transaction, Alert]#Context): Unit = {
    // delete timer
    val timer = timerState.value
    ctx.timerService.deleteProcessingTimeTimer(timer)

    // clean up all states
    timerState.clear()
    flagState.clear()
  }
}