Spark_使用broadcast避免Join出现笛卡尔乘积

package com.atguigu.spark

import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

object TestBroadCast {
def main(args: Array[String]): Unit = {
//1.使用开发工具完成wordcount 创建sc
//1.创建配置文件
val conf = new SparkConf().setMaster(“local[7]”).setAppName(“MyWordCount”)

val sc = new SparkContext(conf)
val rdd1 = sc.makeRDD(List(("a",1),("c",2),("b",2)))
val rdd2 = sc.makeRDD(List(("b",1),("a",2),("c",2)))
//rdd1.join(rdd2)
val map = mutable.Map(("b",1),("a",2),("c",2))
val bc: Broadcast[mutable.Map[String, Int]] = sc.broadcast(map)
rdd1.map{
  case (w,c)=> {
    bc.value.getOrElse(w, 0)
    (w,(c,1))
  }
}.collect().foreach(print)

}
}