Spark–算子实操1
1. TOPn问题
样本:
1516609143867 6 7 64 16
1516609143869 9 4 75 18
1516609143869 1 7 87 12
需求:统计出每一个省份广告被点击次数的TOP3
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object agent2 {
def main(args: Array[String]): Unit = {
//获取上下文对象
val conf = new SparkConf().setMaster("Local[*]").setAppName("agent")
val sc = new SparkContext(conf)
//读取文件创建rdd
val agenrRDD = sc.textFile("in/agent.txt")
//扁平化处理,每次一行
val splitRDD : RDD[((String,String),Int)] = agenrRDD.map { x =>
val split: Array[String] = x.split(" ")
((split(1), split(4)), 1)
}
//分组
//经过扁平化处理之后,数据就变成了((省份,广告),次数)
//聚合操作,经过聚合处理之后,数据就变成了((省份,广告),总次数)
val reduceRDD = splitRDD.reduceByKey(_+_)
//改变结构 (省份,(广告,总次数))便于按照省份分组
val provinceToAdSum : RDD[(String, (String, Int))] = reduceRDD.map(x => (x._1._1, (x._1._2, x._2)))
val provinceGroup = provinceToAdSum.groupByKey()
//排序取出前三
val top3 = provinceGroup.mapValues { x =>
x.toList.sortWith((x, y) => x._2 > y._2).take(3)
}
top3.collect().foreach(println)
}
}
2. WordCount
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object WordCount {
def main(args: Array[String]): Unit = {
//local模式
//创建SparkConf对象
//设定spark计算框架的运行部署环境
val config = new SparkConf().setMaster("local[*]").setAppName("WordCount")
//创建spark上下文对象
val sc = new SparkContext(config)
//读取文件,将文件的内容一行一行的读取出来
//如果是从本地:file:///home/hadoop/apps/spark/in
val lines: RDD[String] = sc.textFile("in/word.txt")
//将一行一行的数据分解为单词
val words: RDD[String] = lines.flatMap(_.split(" "))
//为了统计方便,将数据转化结构为map
val wordToOne: RDD[(String, Int)] = words.map((_, 1))
//将转换之后的数据进行分组聚合
val wordToSune: RDD[(String, Int)] = wordToOne.reduceByKey(_ + _)
//将统计结果采集
val result:Array[(String,Int)]=wordToSune.collect()
result.foreach(f => println(f))
}