package com.meng.nan.day720
import org.apache.log4j.{Level, Logger}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
object SharedVariables {
def main(args: Array[String]): Unit = {
//共享变量之广播变量Broadcast变量
//注意广播变量不适合处理那些大变量
Logger.getLogger(“org.apache.spark”).setLevel(Level.WARN)
Logger.getLogger(“org.apache.hadoop”).setLevel(Level.WARN)
Logger.getLogger(“org.spark_project”).setLevel(Level.WARN)
val conf=new SparkConf()
.setAppName(“SharedVariables”)
.setMaster(“local[*]”)
val sc=new SparkContext(conf)
val list=1 to 10
val listRDD:RDD[Int]=sc.parallelize(list)
//joinOps(sc)
// sc.stop()
acl2Ops(sc)
}
/**
- 使用map或者flatmap来代替join操作
-
join操作使用shuffle的
-
-
使用这种方式可以非常高效的完成类似大小表关联的操作,
-
也就是说将reduce的join---->map的join
- mr中的多表关联:
-
map join
-
reduce join
*/
def joinOps(sc:SparkContext): Unit ={
val stuMap=List(
“1 刘梦男 22 bd-1901-bj”,
“2 常国龙 25 bd-1901-bj”,
“3 张湟熹 24 bd-1901-sz”,
“4 胡盼盼(男) 18 bd-1901-wh”
).map(stuLine=>{
val sid=stuLine.substring(0,1)
val info=stuLine.substring(1).trim
(sid,info)
}).toMap
//创建广播变量
val stuBC:Broadcast[Map[String,String]]=sc.broadcast(stuMap)
//大表
val score=List(
“1 1 math 82”,
“2 1 english 0”,
“3 2 chinese 85.5”,
“4 3 PE 99”,
“5 10 math 99”
)
val scoreRDD:RDD[String]=sc.parallelize(score)
scoreRDD.map(scoreLine=>{
val fields=scoreLine.split("\s+")
val bcMap=stuBC.value
val sid=fields(1)
val info=bcMap.getOrElse(sid,null)
s"KaTeX parse error: Expected 'EOF', got '\t' at position 4: sid\̲t̲info\tKaTeX parse error: Expected 'EOF', got '\t' at position 12: {fields(2)}\̲t̲{fields(3)}"
})
.foreach(println)
}
//累加变量Accumulator
def acl2Ops(sc:SparkContext): Unit ={
//设置累计器
val accu=sc.longAccumulator(“isAccu”)
val list=List(
“hello you hello me”,
“hello you shit me”,
“hello you oh shit”,
“me you hello me”
)
val listRDD=sc.parallelize(list)
val pairs=listRDD.flatMap(.split("\s+"))
.map(word=>{
if (word==“me”||word==“hello”){
accu.add(1)
}
(word,1)
})
val ret=pairs.reduceByKey(+_)
println(accu.value)
println("============")
ret.foreach(println)
println(accu.value)
//再次执行行动算子的话会进行累加,累加是要执行不同操作算子或不同的数据同一个操作算子,才会触发
val ret1=pairs.count()
println("============")
println(accu.value)
println("============")
//将累加器置于零
// accu.reset()
println(accu.value)
println("============")
Thread.sleep(5000)
sc.stop()
}
}