SparkSQL(RDD、DF、DS相互转换)

最新推荐文章于 2023-02-18 08:08:41 发布

球球别报错了啊

最新推荐文章于 2023-02-18 08:08:41 发布

阅读量261

点赞数

文章标签： spark sql 大数据

本文链接：https://blog.csdn.net/qq_41407089/article/details/120900162

版权

object SparkSQL {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("SparkSQL01_Demo")

    //创建SparkSession对象
    val spark: SparkSession = SparkSession.builder().config(conf).getOrCreate()
    import spark.implicits._
    val df: DataFrame = spark.read.json("C:\\BaiduNetdiskDownload\\15.Spark\\2.资料\\spark-3.0.0-bin-hadoop3.2\\spark-3.0.0-bin-hadoop3.2\\spark-3.0.0-bin-hadoop3.2\\bin\\data\\user.json")
    df.show()
    //SQL写法 创建临时表
    df.createOrReplaceTempView("user")
    //查询平均年龄
    spark.sql("select avg(age) from user").show()

    //DSL写法 查询username和age两列
    df.select("username", "age").show()

    //DRR=====>DF

    val rdd: RDD[(String, String, String)] = spark.sparkContext.makeRDD(List(("1", "2", "zhangsan"), ("2", "3", "lisi"), ("3", "4", "wangwu")))
    val frame: DataFrame = rdd.toDF("id", "age", "username")
    frame.show()
    //DF===>DS
    val ds1: Dataset[User] = frame.as[User]
    ds1.show()

    //DS===>DF===>RDD
    val frame1: DataFrame = ds1.toDF()
    val rdd1: RDD[Row] = frame1.rdd
    rdd1.foreach(r => {
      //      获取第一列的
      println(r.getString(0))
    })
    //    RDD===>DS
    val value: Dataset[User] = rdd.map { case (id, age, username) => User(id, age, username) }.toDS()
    //DS===>RDD
    value.rdd
  }

}

case class User(id: String, age: String, username: String)