package com.rdd.topn
import org.apache.spark.rdd.RDD
import org.apache.spark.util.AccumulatorV2
import org.apache.spark.{SparkConf, SparkContext}
import scala.collection.mutable
//使用累加器 拒绝shullfe
object HotCategoryTop10Analysis4 {
def main(args: Array[String]): Unit = {
//TODO Top10热门品类
val conf: SparkConf = new SparkConf().setMaster(“local[*]”).setAppName(“HotCategoryTop10Analysis”)
val sc = new SparkContext(conf)
val actionRDD: RDD[String] = sc.textFile(“in/user_visit_action.txt”)
val acc = new HotCategoryAccumulator
sc.register(acc,“hotCategor”)
actionRDD.foreach {
action => {
val datas = action.split("_")
if (datas(6) != “-1”) {
acc.add(datas(6),“click”)
}
else if (datas(8) != “null”) {
val ids = datas(8).split(",")
ids.foreach(
id=>{
acc.add((id,“order”))
}
)
}
else if (datas(10) != “null”) {
val ids = datas(10).split(",")
ids.foreach(
id=>{
acc.add((id,“pay”))
}
)
}
}
}
val accValu: mutable.Map[String, HotCategory] = acc.value
val categories: mutable.Iterable[HotCategory] = accValu.map((_._2))
val sort: List[HotCategory] = categories.toList.sortWith(
(left, right) => {
if (left.clickCnt > right.clickCnt) {
true
} else if (left.clickCnt == right.clickCnt) {
if (left.orderCnt > right.orderCnt) {
true
} else if (left.orderCnt == right.orderCnt) {
left.payCnt > right.payCnt
} else {
false
}
} else {
false
}
}
)
sort.take(10).foreach(println)
sc.stop()
}
case class HotCategory(cid:String,var clickCnt:Int,var orderCnt:Int,var payCnt:Int)
//自定义累加器
//继承AccumulatorV2定义泛型IN(品类,行为类型)
// OUT:mutable.Map[String,Hotcategory]
class HotCategoryAccumulator extends AccumulatorV2[(String,String),mutable.Map[String,HotCategory]]{
//
private val hcMap: mutable.Map[String, HotCategory] = mutable.MapString,HotCategory
override def isZero: Boolean = {
hcMap.isEmpty
}
override def copy(): AccumulatorV2[(String, String), mutable.Map[String, HotCategory]] = {
new HotCategoryAccumulator()
}
override def reset(): Unit = {
hcMap.clear()
}
//添加数据
override def add(v: (String, String)): Unit = {
val cid: String = v._1
val actionType: String = v._2
val category: HotCategory = hcMap.getOrElse(cid,HotCategory(cid,0,0,0))
if(actionType=="click"){
category.clickCnt+=1
}else if(actionType =="order'"){
category.orderCnt+=1
}else if(actionType=="pay"){
category.payCnt+=1
}
hcMap.update(cid,category)
}
override def merge(other: AccumulatorV2[(String, String), mutable.Map[String, HotCategory]]): Unit = {
val map1: mutable.Map[String, HotCategory] = this.hcMap
val map2: mutable.Map[String, HotCategory] = other.value
map2.foreach{
case (cid,hc)=>{
val category: HotCategory = map1.getOrElse(cid,HotCategory(cid,0,0,0))
category.clickCnt+=hc.clickCnt
category.orderCnt+=hc.orderCnt
category.orderCnt+=hc.payCnt
map1.update(cid,category)
}
}
}
override def value: mutable.Map[String, HotCategory] = hcMap
}
}