spark SQL操作json数据

package hbase

import org.apache.spark.sql.{SaveMode, SparkSession}

/**
* Created by songmingqi on 2018/9/4
* 数据类型
* {“name”:”Signal”,”age”:15}
* {“name”:”May j Lee”,”age”:20}
* {“name”:”Jay Chou”,”age”:36}
* {“name”:”Jack Chen”,”age”:60}
*/
object SparkSQLJson {
case class Person(name:String,age:Long)

def main(args: Array[String]): Unit = {
val spark = SparkSession.builder().appName(“spark json”).master(“local[2]”).getOrCreate()

val df = spark.read.json("file:\\D:\\bigdata\\person.json")
//df.show()
//df.printSchema()
//df.select("name").show()

import spark.implicits._
//df.select($"name",$"age"+2).show()
//df.filter($"age">18).show()
//df.where($"age">20).select("name","age").show()
//df.groupBy("age").count().show()

df.createTempView("Person")
val sql = spark.sql("select * from Person")//.show()

createDataFrameTest(spark)

}
def createDataFrameTest(spark :SparkSession):Unit = {
import spark.implicits._
//创建dateSet，新建一个泛型序列，里面有两个值
val classDS = Seq(Person(“zhao”,22),Person(“song”,25)).toDS()
//classDS.show()
// classDS.write.mode(SaveMode.Overwrite).json(“file:\D:\bigdata\person1.json”)

val primitiveDS = Seq(1,2,3).toDS()
val ds = primitiveDS.map(_+2).collect()
ds.foreach(println(_))

val path = "file:\\D:\\bigdata\\person1.json"
val personDS = spark.read.json(path).as[Person]
personDS.show()

}
}