spark streaming - scala统计hdfs

最新推荐文章于 2024-05-13 02:23:45 发布

贾红平

最新推荐文章于 2024-05-13 02:23:45 发布

阅读量380

点赞数

分类专栏： Spark Scala Streaming系列文章标签： saprk streaming

本文链接：https://blog.csdn.net/qq_18603599/article/details/79970119

版权

Spark Scala Streaming系列专栏收录该内容

4 篇文章 0 订阅

订阅专栏

本文章主要通过spark streaming实现hdfs文件的统计

import org.apache.spark.SparkConf
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds

/**
 * @author jhp
  *         spark streaming读取hdfs文件
 */
object HDFSWordCount {
  
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
        .setMaster("local[2]")  
        .setAppName("HDFSWordCount")
    val ssc = new StreamingContext(conf, Seconds(5))
    
    val lines = ssc.textFileStream("hdfs://spark1:9000/wordcount_dir")  
    val words = lines.flatMap { _.split(" ") }  
    val pairs = words.map { word => (word, 1) }  
    val wordCounts = pairs.reduceByKey(_ + _)  
    
    wordCounts.print()  
    
    ssc.start()
    ssc.awaitTermination()
  }
  
}