Spark——使用广播变量实现wc

package com.rdd.topn

import org.apache.spark.rdd.RDD
import org.apache.spark.util.AccumulatorV2
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

//使用累加器 拒绝shullfe
object HotCategoryTop10Analysis4 {
def main(args: Array[String]): Unit = {
//TODO Top10热门品类
val conf: SparkConf = new SparkConf().setMaster(“local[*]”).setAppName(“HotCategoryTop10Analysis”)
val sc = new SparkContext(conf)
val actionRDD: RDD[String] = sc.textFile(“in/user_visit_action.txt”)
val acc = new HotCategoryAccumulator
sc.register(acc,“hotCategor”)
actionRDD.foreach {
action => {
val datas = action.split("_")
if (datas(6) != “-1”) {
acc.add(datas(6),“click”)
}
else if (datas(8) != “null”) {
val ids = datas(8).split(",")
ids.foreach(
id=>{
acc.add((id,“order”))
}
)
}
else if (datas(10) != “null”) {
val ids = datas(10).split(",")
ids.foreach(
id=>{
acc.add((id,“pay”))
}
)
}
}
}

val accValu: mutable.Map[String, HotCategory] = acc.value
val categories: mutable.Iterable[HotCategory] = accValu.map((_._2))
val sort: List[HotCategory] = categories.toList.sortWith(
  (left, right) => {
    if (left.clickCnt > right.clickCnt) {
      true
    } else if (left.clickCnt == right.clickCnt) {
      if (left.orderCnt > right.orderCnt) {
        true
      } else if (left.orderCnt == right.orderCnt) {
        left.payCnt > right.payCnt
      } else {
        false
      }
    } else {
      false
    }
  }
)
sort.take(10).foreach(println)


sc.stop()

}
case class HotCategory(cid:String,var clickCnt:Int,var orderCnt:Int,var payCnt:Int)
//自定义累加器
//继承AccumulatorV2定义泛型IN(品类，行为类型)
// OUT:mutable.Map[String,Hotcategory]
class HotCategoryAccumulator extends AccumulatorV2[(String,String),mutable.Map[String,HotCategory]]{
//
private val hcMap: mutable.Map[String, HotCategory] = mutable.MapString,HotCategory
override def isZero: Boolean = {
hcMap.isEmpty
}

override def copy(): AccumulatorV2[(String, String), mutable.Map[String, HotCategory]] = {
  new HotCategoryAccumulator()
}

override def reset(): Unit = {
  hcMap.clear()
}
//添加数据
override def add(v: (String, String)): Unit = {
  val cid: String = v._1
  val actionType: String = v._2
  val category: HotCategory = hcMap.getOrElse(cid,HotCategory(cid,0,0,0))
  if(actionType=="click"){
    category.clickCnt+=1
  }else if(actionType =="order'"){
    category.orderCnt+=1
  }else if(actionType=="pay"){
    category.payCnt+=1
  }
  hcMap.update(cid,category)
}

override def merge(other: AccumulatorV2[(String, String), mutable.Map[String, HotCategory]]): Unit = {
  val map1: mutable.Map[String, HotCategory] = this.hcMap
  val map2: mutable.Map[String, HotCategory] = other.value
  map2.foreach{
    case (cid,hc)=>{
      val category: HotCategory = map1.getOrElse(cid,HotCategory(cid,0,0,0))
      category.clickCnt+=hc.clickCnt
      category.orderCnt+=hc.orderCnt
      category.orderCnt+=hc.payCnt
      map1.update(cid,category)
    }
  }
}

override def value: mutable.Map[String, HotCategory] = hcMap

}
}