package com.looc.spark.hpeu
import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.apache.spark.{SparkConf, SparkContext}
object Spark_SQL_DataFrame {
def main(args: Array[String]): Unit = {
// 创建SparkConf()并设置AppName和Master
val conf = new SparkConf().setAppName("SQL").setMaster("local")
// SparkContext依赖SparkConf
val sc = new SparkContext(conf)
// 创建sqlContext
val sqlContext = new SQLContext(sc)
// ----------创建DF的第一种方式----------
// 将personRDD和case class关联
val personRDDOne = sc.textFile("hdfs://mini1:9000/bigdata/person.txt").map(line => {
val fields = line.split(" ")
Person(fields(0).toInt, fields(1), fields(2).toInt)
})
// 导入隐式转换,将RDD转换成DF(DataFrame)
import sqlContext.implicits._
val personDFOne = personRDDOne.toDF()
// ------------------------------------
// ----------创建DF的第二种方式----------
// 从指定的地址创建RDD
val personRDDTwo = sc.textFile("hdfs://mini1:9000/bigdata/person.txt").map(_.split(" "))
// 将RDD转换成DF(DataFrame)
val rowRDD = personRDDTwo.map(line => Row(line(0).toInt, line(1).trim, line(2).toInt))
// 通过StructType直接指定每个字段的schema
val schema = StructType(
List(
StructField("id", IntegerType, true)
,
StructField("name", StringType, true)
,
StructField("age", IntegerType, true)
)
)
// 将schema的信息映射到rowRDD上
val personDFTwo = sqlContext.createDataFrame(rowRDD, schema)
// ------------------------------------
// 将DataFrame注册成为一张person表
personDFOne.registerTempTable("person")
// 传入SQL
sqlContext.sql("select * from person where age >= 25 order by age").show()
// 停止SparkContext
sc.stop()
}
}
// 创建case class
case class Person(id: Int, name: String, age: Int)
Spark SQL DataFrame 小案例
最新推荐文章于 2022-04-12 11:35:18 发布