SparkSQL第一个程序，统计wordCount

最新推荐文章于 2022-07-17 22:21:00 发布

遥遥晚风点点

最新推荐文章于 2022-07-17 22:21:00 发布

阅读量209

点赞数

分类专栏： SparkSQL 文章标签： spark

本文链接：https://blog.csdn.net/Mr_ye931/article/details/108086880

版权

SparkSQL 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}

object SparkSQL01 {

  def main(args: Array[String]): Unit = {

    val spark: SparkSession = SparkSession.builder().appName(this.getClass.getName).master("local[*]").getOrCreate()

    val lines: Dataset[String] = spark.read.textFile("C:\\Users\\LEMMONT\\Desktop\\testdata\\word")

    import spark.implicits._
    val words: Dataset[String] = lines.flatMap(_.split(" "))
    //SQL风格
    words.createTempView("t_words")  //注册视图

    val res: DataFrame = spark.sql("SELECT value,count(1) cou FROM t_words GROUP BY value ORDER BY cou DESC")

    res.show()

    //DSL风格
    val dslRow: Dataset[Row] = words.groupBy($"value" as "word").count()
      .withColumnRenamed("count", "counts") //起别名
      .orderBy($"counts" desc)
    dslRow.show()

    //DSL agg实现
    //要使用聚合函数前一定要导入聚合函数
    import org.apache.spark.sql.functions._
    val dlsRow2: Dataset[Row] = words.groupBy($"value" as "word").agg(count("*").as("counts"))
      .orderBy($"counts" desc)
    dlsRow2.show()

    spark.stop()

  }

}

输出：