Spark中DataSet、DataFrame使用实例

Spark中DataSet使用实例

package nj.zb.kb11

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}

object DataSetDemo {
  case class Point(label:String,x:Double,y:Double)
  case class Category(id:Long,name:String)
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("DataSetDemo")
    val sc: SparkContext = SparkContext.getOrCreate(conf)
    val spark: SparkSession = SparkSession.builder().master("local[*]").appName("dsDemo")
      .getOrCreate()
    import spark.implicits._
//    val ds1: Dataset[Int] = spark.createDataset(1 to 6)
//    ds1.show()
//    val ds2: Dataset[(String, Int)] = spark.createDataset(List(("a",1),("b",2)))
//    ds2.show()
//    val ds3: Dataset[(String, Int, Int)] = spark.createDataset(sc.parallelize(List(("gree",37,170),("gll",18,180))))
//    ds3.show()

    val points = Seq(Point("njzb",23.23,48.74),Point("njnz",26.12,48.77))

    val pointsDS: Dataset[Point] = points.toDS()    //直接转换成  dataset类型
//    pointsDS.printSchema()

    val categories = Seq(Category(1,"njzb"),Category(2,"njnz"))
    val categoriesDS: Dataset[Category] = categories.toDS()
//    categoriesDS.printSchema()

//   val df: DataFrame = pointsDS.join(categoriesDS,pointsDS("label")===categoriesDS("name"))
//    df.printSchema()
//    df.show()

//
//    val pointsRDD: RDD[(String, Double, Double)] = sc.parallelize(List(("njzb",23.23,48.74),("njnz",26.12,48.77)))
//    val categoriesRDD: RDD[(Int, String)] = sc.parallelize(List((1,"njzb"),(2,"njnz")))
//    val pointsDS: Dataset[Point] = pointsRDD.map(x=>Point(x._1,x._2,x._3)).toDS()
//    val categoryDS: Dataset[Category] = categoriesRDD.map(x=>Category(x._1,x._2)).toDS()
//
  }
}

Spark中DataFrame使用实例

package nj.zb.kb11


import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spark.sql.types._


object DataFrameDemo {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[*]").setAppName("sparkreadcsv")
    val sc = SparkContext.getOrCreate(conf)
    val spark: SparkSession = SparkSession.builder().appName("DataFrameDemo").master("local[*]").getOrCreate()

    import spark.implicits._
    val people: RDD[String] = sc.textFile("in/people.txt")
    val schemaString="id name age"
    val schema: StructType = StructType(schemaString.split(" ").map(x=>StructField(x,StringType,true)))

    val fields = Array(
      StructField("id", IntegerType, true),
      StructField("name", StringType, true),
      StructField("age", IntegerType, true)
    )



    val row: RDD[Row] = people.map(x =>
      x.split(" ")).map(x => Row(x(0), x(1), x(2)))


    val peopleDF: DataFrame = spark.createDataFrame(row,schema)
    peopleDF.printSchema()
    peopleDF.show()

    peopleDF.createOrReplaceTempView("people")
    val a: DataFrame = spark.sql("select name from people where age > 35")
    a.show()

    //将dataFrame以parquet的格式存入到指定目录
//    peopleDF.write.parquet("out/parquettest")

    //将parquet格式的内容读取到spark程序中
    val frame1: DataFrame = spark.read.parquet("out/parquettest")
    frame1.printSchema()
    frame1.show()

  }
}