SparkSQL实现WC

最新推荐文章于 2022-06-23 17:44:17 发布

Dale_w_code

最新推荐文章于 2022-06-23 17:44:17 发布

阅读量225

点赞数

分类专栏： spark

本文链接：https://blog.csdn.net/qq_37240582/article/details/86543375

版权

spark 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spark.sql.types.{StringType, StructField, StructType}

object SparkSqlWC {
  def main(args: Array[String]): Unit = {
    val spark: SparkSession = SparkSession.builder().appName("WC").master("local[2]").getOrCreate()
    import spark.implicits._
    val schema: StructType = new StructType(Array(StructField("field",StringType,true)))
    val sc = spark.sparkContext
    val rdd1 = sc.textFile("i://test.txt").flatMap(_.split(","))
    //sparkSQL实现WC
    val rdd = rdd1.map(field => {Row(field)})

    
    val df: DataFrame = spark.createDataFrame(rdd,schema)
    df.groupBy("field").count().show()

    println("spark core实现wc")
    println(rdd1.map((_,1)).reduceByKey(_+_).collect().toBuffer)
    spark.stop()
  }
}