spark sql 测试小程序

          这一篇是对上一篇造数据脚本的应用，造出的数据大概3.5G左右，然后再intellj idea 中跑了一下，效果不错，当然只能是本地跑了。

下为代码：

package main.scala.UserPlatformCount


import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spark.sql.types._
import org.apache.spark.{SparkConf, SparkContext}
import org.dmg.pmml.FieldName



/**
  * Created by George on 2016/12/9.
  */
object PlatformInfoCounter {
  private val datapath = "E:/platform.txt"
  private val SchemaString = "id,name,age,phoneType,clicks,logintime"
  def main(args: Array[String]): Unit = {
    if (args.length < 1){
      args(0)= datapath
    }
    System.exit(1)
  }
  // set a environment of sparkSql
  val conf = new SparkConf().setAppName("platform").setMaster("local[4]")
  val sc = new SparkContext(conf)
  val sqlctx = new org.apache.spark.sql.SQLContext(sc)

  import sqlctx.implicits._
  val platfromdata = sc.textFile(datapath)
  val SchemaArray = SchemaString.split(",")
  val Schema = StructType(SchemaArray.map(fieldname => StructField(fieldname,StringType,true)))
  val rowRDD :RDD[Row] = platfromdata.map(_.split(",")).map(row => Row(row(0),row(1).trim,row(2).trim,row(3).trim,row(4).trim,row(5).trim))
  val platFormDF = sqlctx.createDataFrame(rowRDD,Schema)
  platFormDF.registerTempTable("PlatInfo")

  //use sparksql to count
  val phonetype = sqlctx.sql("select name,phoneType,count(1) from PlatInfo group by name,phoneType limit 15")
  phonetype.show()

  val clicks = sqlctx.sql("select name,phoneType,age,clicks,count(1) from PlatInfo group by name,phoneType,age,clicks limit 20")
  clicks.sort().show()

}
下为测试结果：