这一篇是对上一篇造数据脚本的应用,造出的数据大概3.5G左右,然后再intellj idea 中跑了一下,效果不错,当然只能是本地跑了。
下为代码:
package main.scala.UserPlatformCount import org.apache.spark.rdd.RDD import org.apache.spark.sql.{Row, SQLContext} import org.apache.spark.sql.types._ import org.apache.spark.{SparkConf, SparkContext} import org.dmg.pmml.FieldName /** * Created by George on 2016/12/9. */ object PlatformInfoCounter { private val datapath = "E:/platform.txt" private val SchemaString = "id,name,age,phoneType,clicks,logintime" def main(args: Array[String]): Unit = { if (args.length < 1){ args(0)= datapath } System.exit(1) } // set a environment of sparkSql val conf = new SparkConf().setAppName("platform").setMaster("local[4]") val sc = new SparkContext(conf) val sqlctx = new org.apache.spark.sql.SQLContext(sc) import sqlctx.implicits._ val platfromdata = sc.textFile(datapath) val SchemaArray = SchemaString.split(",") val Schema = StructType(SchemaArray.map(fieldname => StructField(fieldname,StringType,true))) val rowRDD :RDD[Row] = platfromdata.map(_.split(",")).map(row => Row(row(0),row(1).trim,row(2).trim,row(3).trim,row(4).trim,row(5).trim)) val platFormDF = sqlctx.createDataFrame(rowRDD,Schema) platFormDF.registerTempTable("PlatInfo") //use sparksql to count val phonetype = sqlctx.sql("select name,phoneType,count(1) from PlatInfo group by name,phoneType limit 15") phonetype.show() val clicks = sqlctx.sql("select name,phoneType,age,clicks,count(1) from PlatInfo group by name,phoneType,age,clicks limit 20") clicks.sort().show() } 下为测试结果: