sparksql php,sparkSql 第一个demo

package com.ws.sparksql

import org.apache.spark.rdd.RDD

import org.apache.spark.sql.{DataFrame, SQLContext}

import org.apache.spark.{SparkConf, SparkContext}

/**

* spark sql版本 1.1x

* spark sql 第一个demo

*/

object SparkSqlDemo {

def main(args: Array[String]): Unit = {

val conf = new SparkConf().setAppName("SparkSqlDemo").setMaster("local[2]")

val sc = new SparkContext(conf)

//创建一个sparksql实例

val sQLContext = new SQLContext(sc)

val dataRdd: RDD[String] = sc.textFile("hdfs://hadoop-01:9000/student")

val studentRdd: RDD[Student] = dataRdd.map(line => {

val strArr: Array[String] = line.split(",")

val id = strArr(0).toLong

val name = strArr(1)

val age = strArr(2).toInt

val score = strArr(3).toInt

Student(id, name, age, score)

})

import sQLContext.implicits._

//把普通RDD转换成特殊的RDD，变成dataFrame

//个人觉得dataFrame和mysql表结构一样，其中的一条数据就是一个pojo，pojo映射成mysql的表结构

val sdf: DataFrame = studentRdd.toDF

//将dataFrame注册一个表

sdf.registerTempTable("t_student")

//写sprak sql,本质也是Transformation

val result: DataFrame = sQLContext.sql("select * from t_student order by score desc , age asc")

//触发Action操作，展示结果

result.show()

sc.stop()

}

}

case class Student(id: Long, name: String, age: Int, score: Int)

结果 ：

+---+----+---+-----+

| id|name|age|score|

+---+----+---+-----+

| 1| 张三| 18| 150|

| 2| 李四| 19| 150|

| 3| 王五| 20| 98|

| 4| 赵六| 17| 88|

+---+----+---+-----+