spark写sql语句范例

//创建一个sparkSession对象
val sparkSession = SparkSession.builder()
  .appName("createDF2")
  .master("local")
  .getOrCreate()
val rdd = sparkSession.sparkContext.textFile("c://data/person.txt")
//整理数据，ROW类型
val rowrdd = rdd.map(line=>{
  val fields = line.split(",")
  val id = fields(0).toLong
  val name = fields(1)
  val age = fields(2).toInt
  val facevalue = fields(3).toDouble
  Row(id,name,age,facevalue)
})

//scheme:定义DataFrame里面元素的数据类型，以及对每个元素的约束
val structType = StructType(List(
  StructField("id",LongType,true),
  StructField("name",StringType,true),
  StructField("age",IntegerType,true),
  StructField("facevalue",DoubleType,true)
))
//将rowrdd和structType关联
val df:DataFrame = sparkSession.createDataFrame(rowrdd,structType)
//创建一个视图
df.createOrReplaceTempView("Tperson")
//基于注册的视图写SQL
val res:DataFrame = sparkSession.sql("SELECT name,age FROM Tperson ORDER BY age asc")

res.show()

sparkSession.stop()