IDEA 开发 SparkSQL-配置与简单使用

1.添加依赖

在idea项目的pom.xml中添加依赖。

<!--spark sql依赖，注意版本号-->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-sql_2.12</artifactId>
    <version>3.0.0</version>
</dependency>

2.案例代码

package com.zf.bigdata.spark.sql

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}

object Spark01_SparkSql_Basic {

    def main(args: Array[String]): Unit = {

        //创建上下文环境配置对象
        val sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparkSql")
        //创建 SparkSession 对象
        val spark = SparkSession.builder().config(sparkConf).getOrCreate()

        // DataFrame
        val df: DataFrame = spark.read.json("datas/user.json")
        //df.show()

        // DataFrame => Sql

        //df.createOrReplaceTempView("user")
        //spark.sql("select * from user").show()
        //spark.sql("select age from user").show()
        //spark.sql("select avg(age) from user").show()

        //DataFrame => Dsl

        //如果涉及到转换操作，转换需要引入隐式转换规则，否则无法转换，比如使用$提取数据的值
        //spark 不是包名，是上下文环境对象名
        import spark.implicits._
        //df.select("age","username").show()
        //df.select($"age"+1).show()
        //df.select('age+1).show()

        // DataSet

        //val seq = Seq(1,2,3,4)
        //val ds: Dataset[Int] = seq.toDS()
        // ds.show()

        // RDD <=> DataFrame
        val rdd = spark.sparkContext.makeRDD(List((1,"张三",10),(2,"李四",20)))
        val df1: DataFrame = rdd.toDF("id", "name", "age")
        val rdd1: RDD[Row] = df1.rdd

        // DataFrame <=> DataSet
        val ds: Dataset[User] = df1.as[User]
        val df2: DataFrame = ds.toDF()

        // RDD <=> DataSet
        val ds1: Dataset[User] = rdd.map {
            case (id, name, age) => {
                User(id, name = name, age = age)
            }
        }.toDS()
        val rdd2: RDD[User] = ds1.rdd




        spark.stop()
    }
    case class User(id:Int,name:String,age:Int)

}