Scala反射创建DataFrame（方式一，已知数据类型）

package sparkSql.方法1创建DataFrame

import org.apache.spark.sql.{SQLContext, SaveMode}
import org.apache.spark.{SparkConf, SparkContext}

object Student {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName(this.getClass.getName).setMaster("local")
val sc = new SparkContext(conf)
val sQLContext = new SQLContext(sc)

val file = sc.textFile("C:\\Program Files\\feiq\\Recv Files\\day11-Spark SQL\\students.txt")

val stuRDD = file.map(t=>{
val arr = t.split(",")
Stu(arr(0).toInt,arr(1),arr(2).toInt);
 })

import sQLContext.implicits._
val df = stuRDD.toDF()
 df.registerTempTable("stu")
val df2 = sQLContext.sql("select name from stu where age < 18")
//df2.show()
// 保存数据类型
// df2.write.orc()
// df2.write.json()
// df2.write.text()

 df2.write.format("text").save("D:\\stuTest")//只能一个字段
 //df2.write.mode(SaveMode.Append).save("D:\\stuTest")//只能一个字段


 // df.rdd 转javaRDD
// df.show()
 }
}
case class Stu(id:Int,name:String,age:Int)

pom.xml文件同java上面

转载于:https://www.cnblogs.com/VIP8/p/10434368.html