Flink window Function - ReduceWordCount

最新推荐文章于 2024-11-28 20:48:51 发布

weixin_30375247

最新推荐文章于 2024-11-28 20:48:51 发布

阅读量136

点赞数

文章标签：大数据 scala

原文链接：http://www.cnblogs.com/maoxiangyi/p/10977926.html

版权

package window

import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.api.scala._

/**
 * @author: create by maoxiangyi
 * @version: v1.0
 * @description: window
 * @date:2019 /6/4
 */
object ReduceWordCount {
 def main(args: Array[String]): Unit = {
  //设置环境
  val env: StreamExecutionEnvironment = StreamExecutionEnvironment.createLocalEnvironment()
  //设置数据源
  env.addSource(new SourceFunction[String] {
   override def run(ctx: SourceFunction.SourceContext[String]): Unit = {
    while (true) {
     ctx.collect("hello hadoop hello storm hello spark")
     Thread.sleep(1000)
    }
   }

   override def cancel(): Unit = {}
  })
   //计算逻辑
   .flatMap(_.split(" "))
   .map((_, 1))
   .keyBy(_._1)
   .timeWindow(Time.seconds(10), Time.seconds(10))
   .reduce((x, y) => {
    (x._1, x._2 + y._2)
   })
   //输出结果
   .addSink(x => {
   print(x + "\n")
  })
  //提交任务
  env.execute("word count")
 }
}