import org.apache.spark.{SparkConf, SparkContext}
import scala.collection.mutable.ArrayBuffer
/**
* Created by yz02 on 2017/6/15.
*/
object T_coalesce {
System.setProperty("hadoop.home.dir","F:\\hadoop-2.6.5")
def fun_index(index : Int, iter : Iterator[String]) : Iterator[String] = {
var list = ArrayBuffer[String]()
while (iter.hasNext)
{
val name : String = iter.next()
var fs = index + ":" + name
list += fs
// println(fs)
}
return list.iterator
}
def fun_index1(index : Int, iter : Iterator[String]) : Iterator[String] = {
var list = ArrayBuffer[String]()
while (iter.hasNext)
{
val name : String = iter.next()
var fs = index + ":" + name
list += fs
println(fs)
}
return list.iterator
}
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("coalesce_test").setMaster("local")
val sc = new SparkContext(conf)
//coalesce算子,功能是将RDD的partition的数量减缩,减少
//将一定的数据压缩到更少的partition分区中去
//使用场景!很多时候在filter算子之后会优化一下使用coalesce算子
//filter算子应用到RDD上面,说白了会应用到RDD对应的里面的每一个partition上去
//数据倾斜,换句话说就是有可能有的partition里面就剩下了一条数据!!
//建议使用coalesce算子,从而让各个partition中的数据都更加紧凑!
val nameList : List[String] = List("liupeng1", "liupeng2", "liuipeng3",
"liupeng4", "liupeng5", "liupeng6",
"liupeng7", "liupeng8", "liupeng9",
"liupeng10", "liupeng11", "liupeng12"
)
val nameRDD = sc.parallelize(nameList, 6)
val nameRDD2 = nameRDD.mapPartitionsWithIndex(fun_index)
val nameRDD3 = nameRDD2.coalesce(3)
val nameRDD4 = nameRDD3.mapPartitionsWithIndex(fun_index1)
val info : Array[String] = nameRDD4.collect()
}
}
运行结果:
0:0:liupeng1
0:0:liupeng2
0:1:liuipeng3
0:1:liupeng4
0:0:liupeng2
0:1:liuipeng3
0:1:liupeng4
1:2:liupeng5
1:2:liupeng6
1:3:liupeng7
1:3:liupeng8
1:2:liupeng6
1:3:liupeng7
1:3:liupeng8
2:4:liupeng9
2:4:liupeng10
2:5:liupeng11
2:5:liupeng12
2:4:liupeng10
2:5:liupeng11
2:5:liupeng12