Spark阶段案例分组TopN

数据源：

http://bigdata.zpark.cn/laozhang
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laoduan
http://bigdata.zpark.cn/laoduan
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laoduan
http://bigdata.zpark.cn/laoduan
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laoduan
http://bigdata.zpark.cn/laoduan
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://php.zpark.cn/laoli
http://php.zpark.cn/laoliu
http://php.zpark.cn/laoli
http://php.zpark.cn/laoli

MostJobPerson

import java.net.URL

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object MostJobPerson {
  def main(args: Array[String]): Unit = {
    //初始化
    val conf = new SparkConf()
    conf.setAppName("MostJobPerson")
    conf.setMaster("local")
    val sc = new SparkContext(conf)
    //读文件
    val lines: RDD[String] = sc.textFile("C:\\Users\\S\\Desktop\\内民大实训\\person.log")
    //map():第一次处理数据，把job和person当成key  1当成value
    val jobPersonAndOne: RDD[((String, String), Int)] = lines.map(line => {
      val i: Int = line.lastIndexOf("/")
      val person: String = line.substring(i + 1)
      val urlString: String = line.substring(0, i)
      val url = new URL(urlString)
      val host: String = url.getHost
      val strings: Array[String] = host.split("\\.")
      val job: String = strings(0)
      ((job, person), 1)
    })
    //聚合，把key（job+person）相同的数据聚合到一起，value值累加了
//    value.reduceByKey(_+_)
    val reduced: RDD[((String, String), Int)] = jobPersonAndOne.reduceByKey((x,y) => x+y)
    //
    //分组：_带表((javaee,laoyang),9)，_1代表(javaee,laoyang),_1代表javaee,
    val grouped: RDD[(String, Iterable[((String, String), Int)])] = reduced.groupBy(_._1._1)

    //key不变，对没有个value  Iterable[((String, String), Int)])进行处理
    //调用.toList把Iterable迭代器转成List
    val mapval: RDD[(String, List[((String, String), Int)])] = grouped.mapValues(x => {
      val list: List[((String, String), Int)] = x.toList
      print(list)
      /*
      List(((javaee,xiaoxu),6), ((javaee,laoyang),9))
      List(((php,laoliu),1), ((php,laoli),3))
      List(((bigdata,laozhang),1), ((bigdata,laozhao),15), ((bigdata,laoduan),6))
      */
      //_代表：((javaee,xiaoxu),6)  ._2代表6   .reverse:倒序
      val sorted: List[((String, String), Int)] = list.sortBy(_._2).reverse
       // .take(3)作用取出前三条
      val tuples: List[((String, String), Int)] = sorted.take(3)
      //不要遗漏最后的sorted，作用是向下传递
      tuples
    })

//    val sorted: RDD[((String, String), Int)] = reduced.sortBy(_._2,false)
    //collect:满世界去收集
   val tuples: Array[(String, List[((String, String), Int)])] = mapval.collect()

    print(tuples.toBuffer)
    sc.stop()
  }
}