sparksql

pom

<dependency>

<groupId>org.apache.spark</groupId>

<artifactId>spark-sql_2.11</artifactId>

<version>2.3.0</version>

<scope>provided</scope>

</dependency>

初始化 

    val spark = SparkSession

                  .builder()

                  .appName("topn")

                  .master("local[2]")

                  .getOrCreate()     

rdd转DataFrame 

    val df = rddToDataFrame(spark)

    df.printSchema()

    df.select("_1", "_2").show() 

df.take(3).foreach(println)



  def rddToDataFrame(spark:SparkSession):DataFrame = {    

    // 隐式转换

      import spark.implicits._    

    //读取数据

    val fileRDD = spark.sparkContext.textFile("D:\\file\\wc.txt")    

    val wordCountsRDD = fileRDD.flatMap(_.split("\\s+"))

                            .map((_,1))

                            .reduceByKey(_+_)

                            .map(x =>(x._2,x._1))

                            .sortByKey(false)

                            .map(x =>(x._2,x._1))

                            

      //rdd转DataFrame              

      val df = wordCountsRDD.toDF()   

      df

  }

##

root

 |-- _1: string (nullable = true)

 |-- _2: integer (nullable = false)



+---------+---+

|       _1| _2|

+---------+---+

|    spark| 14|

|    storm|  6|

|    scala|  4|

|       es|  3|

|    mysql|  3|

|     solr|  2|

|     hdfs|  2|

|    hbase|  2|

|     hive|  1|

|     mlib|  1|

|    oozie|  1|

|   python|  1|

|mapreduce|  1|

|    flink|  1|

|    linux|  1|

|     java|  1|

|  azkaban|  1|

|    kafka|  1|

|   hadoop|  1|

|    flume|  1|

+---------+---+



[spark,14]

[storm,6]

[scala,4]

 

rdd 转DataSet

      val ds =rddToDataSet(spark)

      ds.createOrReplaceTempView("wordcount")//创建一个本地临时视图

      spark.sql("select word,count(1) as count from wordcount group by word order by count desc limit 3").show()



  case class wordcount(word:String,count:Int)

  def rddToDataSet(spark:SparkSession):Dataset[wordcount] ={

    // For implicit conversions like converting RDDs to DataFrames

      import spark.implicits._

    //读取数据

    val ds = spark

              .sparkContext

              .textFile("D:\\file\\wc.txt")

              .flatMap(_.split("\\s+"))

              .map((_,1))

              .map(x =>(wordcount(x._1,x._2))).toDS()

    ds

  }



##

+-----+-----+

| word|count|

+-----+-----+

|spark|   14|

|storm|    6|

|scala|    4|

+-----+-----+

DataSet转DataFrame

val df = dataSetToDataFrame(spark)
      df.select("word", "count").groupBy("word").count().show()
      val df2= df.select("word", "count").groupBy("word").count() 
      df2.orderBy(df2("count").desc).show
      val df3 = df2.orderBy(df2("count").desc) 
      df3.take(3).foreach(println)

  def dataSetToDataFrame(spark:SparkSession):DataFrame = {
    // For implicit conversions like converting RDDs to DataFrames
      import spark.implicits._
    //读取数据
    val ds = spark
              .read
              .textFile("D:\\file\\wc.txt")
              .flatMap(_.split("\\s+"))
              .map((_,1))
   
    val df=ds.toDF("word","count")
    df
  }

##
+---------+-----+
|     word|count|
+---------+-----+
|    hbase|    2|
|    kafka|    1|
|     mlib|    1|
|    mysql|    3|
|       es|    3|
|    scala|    4|
|    spark|   14|
|    storm|    6|
|    flume|    1|
|    linux|    1|
|mapreduce|    1|
|     solr|    2|
|    flink|    1|
|    oozie|    1|
|     hdfs|    2|
|     java|    1|
|   hadoop|    1|
|  azkaban|    1|
|   python|    1|
|     hive|    1|
+---------+-----+


+---------+-----+
|     word|count|
+---------+-----+
|    spark|   14|
|    storm|    6|
|    scala|    4|
|    mysql|    3|
|       es|    3|
|    hbase|    2|
|     solr|    2|
|     hdfs|    2|
|    kafka|    1|
|    flume|    1|
|     mlib|    1|
|   hadoop|    1|
|    linux|    1|
|    flink|    1|
|     java|    1|
|  azkaban|    1|
|     hive|    1|
|mapreduce|    1|
|    oozie|    1|
|   python|    1|
+---------+-----+


[spark,14]
[storm,6]
[scala,4]

DataSet转RDD 

        val fileRDD = dataSetToRDD(spark)
        val wordCounts = fileRDD.flatMap(_.split("\\s+"))
                            .map((_,1))
                            .reduceByKey(_+_)
                            .map(x =>(x._2,x._1))
                            .sortByKey(false)
                            .map(x =>(x._2,x._1))
        wordCounts.collect().foreach(println)
         wordCounts.take(3).foreach(println)    

  def dataSetToRDD(spark:SparkSession)={
    val rdd = spark.read
                  .textFile("D:\\file\\wc.txt")
                  .rdd
    rdd
  }
##
(spark,14)
(storm,6)
(scala,4)
(es,3)
(mysql,3)
(solr,2)
(hdfs,2)
(hbase,2)
(hive,1)
(mlib,1)
(hadoop,1)
(oozie,1)
(flume,1)
(python,1)
(mapreduce,1)
(flink,1)
(linux,1)
(java,1)
(azkaban,1)
(kafka,1)

(spark,14)
(storm,6)
(scala,4)

DataFrame转RDD 

 val jsonRDD = dataFrameToRDD(spark)

 jsonRDD.foreach(println)      



  def dataFrameToRDD(spark:SparkSession)={

    import spark.implicits._

    val people = spark.read

                  .json("D:\\file\\people.json")   

    people.printSchema()

    people.createOrReplaceTempView("people")   

    val teenagers =spark.sql("select name from people where age >=13 and age<=19")

    teenagers.rdd

  }





{"name":"Michael", "age":28}

{"name":"Andy", "age":30}

{"name":"Justin", "age":19}



root

 |-- age: long (nullable = true)

 |-- name: string (nullable = true)



[Justin]