spark sql-CSDN博客

本文链接：https://blog.csdn.net/u010220089/article/details/49658437

spark sql

package com.spark.sql

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext

object JSONDataSource {
  def main(args: Array[String]): Unit = {
     val conf = new SparkConf()
        .setAppName("JSONDataSource")
        .setMaster("local")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)
      // 创建学生成绩DataFrame
    val studentScoresDF = sqlContext.read.json("E:\\spark-1.4.1-bin-hadoop2.4\\examples\\src\\main\\resources\\people.json")
   // 查询出分数大于80分的学生成绩信息，以及学生姓名
   studentScoresDF.registerTempTable("student_scores")
    val goodStudentScoresDF = sqlContext.sql("select name,age from student_scores where age>=19")
    val goodStudentNames = goodStudentScoresDF.rdd.map { row => row(0) }.collect()
    goodStudentNames.map { row => println("-----name----->"+row) }
    val goodStudentNamesAges = goodStudentScoresDF.rdd.map { row => (row(0),row(0)) }.collect()
      goodStudentNamesAges.map { row => println("-----name and age----->"+row) }
   // println(goodStudentNames)
   
  }
}