Flink实现TopN URL访问量统计

最新推荐文章于 2025-05-17 15:50:19 发布

渣渣盟

最新推荐文章于 2025-05-17 15:50:19 发布

阅读量920

点赞数 25

分类专栏：大数据技术与应用 # Flink # 基于Scala对Flink的实现文章标签： flink 大数据 scala

本文链接：https://blog.csdn.net/m0_57376564/article/details/147991040

版权

大数据技术与应用同时被 3 个专栏收录

22 篇文章

订阅专栏

基于Scala对Flink的实现

16 篇文章

订阅专栏

Flink

13 篇文章

订阅专栏

#新星杯·14天创作挑战营·第11期#

package processfunction

import org.apache.flink.api.common.functions.AggregateFunction
import org.apache.flink.api.common.state.{ListState, ListStateDescriptor}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.scala.function.ProcessWindowFunction
import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.streaming.api.windowing.windows.TimeWindow
import org.apache.flink.util.Collector
import org.apache.hadoop.metrics2.util.Metrics2Util.TopN
import source.{ClickSource, Event}

import scala.collection.convert.ImplicitConversions.`iterable AsScalaIterable`

case class UrlViewCount(url:String,count:Long,widowStart:Long,windowEnd:Long)
/**
 *
 * @PROJECT_NAME: flink1.13
 * @PACKAGE_NAME: processfunction
 * @author: 赵嘉盟-HONOR
 * @data: 2023-11-24 21:55
 * @DESCRIPTION
 *
 */
object TopNKeyedProcessFunction {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)

    val data = env.addSource(new ClickSource).assignAscendingTimestamps(_.timestamp)

    val urlCountStream = data.keyBy(_.url)
      .window(SlidingEventTimeWindows.of(Time.seconds(10), Time.seconds(5)))
      .aggregate(new UrlViewCountAgg, new UrlViewCountResult)

    urlCountStream.keyBy(_.windowEnd).process(new TopN(5)).print()

    env.execute("TopNDemo2")
  }
  class UrlViewCountAgg extends AggregateFunction[source.Event,Long,Long] {
    override def createAccumulator(): Long = 0L
    override def add(in: Event, acc: Long): Long = acc+1
    override def getResult(acc: Long): Long = acc
    override def merge(acc: Long, acc1: Long): Long = ???
  }
  class UrlViewCountResult extends ProcessWindowFunction[Long,UrlViewCount,String,TimeWindow] {
    override def process(key: String, context: Context, elements: Iterable[Long], out: Collector[UrlViewCount]): Unit = {
      out.collect(UrlViewCount(
        key,elements.iterator.next(),context.window.getStart,context.window.getEnd
      ))
    }
  }
  class TopN(topN:Int) extends KeyedProcessFunction[Long,UrlViewCount,String] {
    var urlViewCountListState:ListState[UrlViewCount]=_
    override def open(parameters: Configuration): Unit = {
      urlViewCountListState = getRuntimeContext.getListState(new ListStateDescriptor[UrlViewCount]("list-state", classOf[UrlViewCount]))
    }
    override def processElement(i: UrlViewCount, context: KeyedProcessFunction[Long, UrlViewCount, String]#Context, collector: Collector[String]): Unit = {
      urlViewCountListState.add(i)
      context.timerService().registerEventTimeTimer(i.windowEnd+1)
    }
    override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[Long, UrlViewCount, String]#OnTimerContext, out: Collector[String]): Unit = {
      val topNList = urlViewCountListState.get().toList.sortBy(-_.count).take(topN)
      val builder = new StringBuilder()
      builder.append(s"========窗口：${timestamp-1-10000} ~ ${timestamp-1} ======= \n")
      for (i <- topNList.indices){
        val urlViewCount = topNList(i)
        builder.append(
          s"浏览量Top ${i+1} " +
          s"url: ${urlViewCount.url} " +
          s"浏览量是： ${urlViewCount.count} \n")
      }
      out.collect(builder.toString())
    }
  }
}

这段代码展示了如何使用 Apache Flink 实现一个 TopN 统计 的功能，即统计某个时间窗口内访问量最高的前 N 个 URL。以下是代码的详细解释和背景知识拓展。

代码解释

1. 环境设置

val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)

StreamExecutionEnvironment.getExecutionEnvironment：获取流处理执行环境。
env.setParallelism(1)：设置并行度为 1，方便调试和观察结果。

2. 数据源与时间戳分配

val data = env.addSource(new ClickSource).assignAscendingTimestamps(_.timestamp)

addSource(new ClickSource)：从自定义数据源 ClickSource 读取数据。
assignAscendingTimestamps(_.timestamp)：为数据分配时间戳，用于事件时间处理。

3. 窗口统计

val urlCountStream = data.keyBy(_.url)
  .window(SlidingEventTimeWindows.of(Time.seconds(10), Time.seconds(5)))
  .aggregate(new UrlViewCountAgg, new UrlViewCountResult)

keyBy(_.url)：按 URL 分组。
window(SlidingEventTimeWindows.of(Time.seconds(10), Time.seconds(5)))：定义滑动窗口，窗口大小为 10 秒，滑动步长为 5 秒。
aggregate(new UrlViewCountAgg, new UrlViewCountResult)：使用聚合函数 UrlViewCountAgg 和窗口函数 UrlViewCountResult 统计每个 URL 的访问量。

4. 聚合函数

class UrlViewCountAgg extends AggregateFunction[source.Event, Long, Long] {
  override def createAccumulator(): Long = 0L
  override def add(in: Event, acc: Long): Long = acc + 1
  override def getResult(acc: Long): Long = acc
  override def merge(acc: Long, acc1: Long): Long = ???
}

createAccumulator：初始化累加器。
add：对每条数据进行累加。
getResult：返回累加结果。
merge：合并累加器（未实现）。

5. 窗口函数

class UrlViewCountResult extends ProcessWindowFunction[Long, UrlViewCount, String, TimeWindow] {
  override def process(key: String, context: Context, elements: Iterable[Long], out: Collector[UrlViewCount]): Unit = {
    out.collect(UrlViewCount(key, elements.iterator.next(), context.window.getStart, context.window.getEnd))
  }
}

process：将聚合结果封装为 UrlViewCount 对象，包含 URL、访问量、窗口开始时间和窗口结束时间。

6. TopN 统计

urlCountStream.keyBy(_.windowEnd).process(new TopN(5)).print()

keyBy(_.windowEnd)：按窗口结束时间分组。
process(new TopN(5))：使用 TopN 函数统计每个窗口内访问量最高的前 5 个 URL。
print：打印结果。

7. TopN 函数

class TopN(topN: Int) extends KeyedProcessFunction[Long, UrlViewCount, String] {
  var urlViewCountListState: ListState[UrlViewCount] = _
  override def open(parameters: Configuration): Unit = {
    urlViewCountListState = getRuntimeContext.getListState(new ListStateDescriptor[UrlViewCount]("list-state", classOf[UrlViewCount]))
  }
  override def processElement(i: UrlViewCount, context: KeyedProcessFunction[Long, UrlViewCount, String]#Context, collector: Collector[String]): Unit = {
    urlViewCountListState.add(i)
    context.timerService().registerEventTimeTimer(i.windowEnd + 1)
  }
  override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[Long, UrlViewCount, String]#OnTimerContext, out: Collector[String]): Unit = {
    val topNList = urlViewCountListState.get().toList.sortBy(-_.count).take(topN)
    val builder = new StringBuilder()
    builder.append(s"========窗口：${timestamp - 1 - 10000} ~ ${timestamp - 1} ======= \n")
    for (i <- topNList.indices) {
      val urlViewCount = topNList(i)
      builder.append(
        s"浏览量Top ${i + 1} " +
        s"url: ${urlViewCount.url} " +
        s"浏览量是： ${urlViewCount.count} \n")
    }
    out.collect(builder.toString())
  }
}