java coalesce_java spark算子repartition，coalesce

/*** # _*_ coding:utf-8 _*_

* # Author:xiaoshubiao

* # Time : 2020/5/13 14:13

* java map,mapfunctions

**/

importorg.apache.spark.SparkConf;importorg.apache.spark.api.java.JavaRDD;importorg.apache.spark.api.java.JavaSparkContext;importorg.apache.spark.api.java.function.FlatMapFunction;importorg.apache.spark.api.java.function.Function2;importjava.util.ArrayList;importjava.util.Arrays;importjava.util.Iterator;importjava.util.List;public classspark_function {public static voidmain(String[] args) {

SparkConf conf= new SparkConf().setMaster("local[*]").setAppName("spark_java_function");

JavaSparkContext sc= newJavaSparkContext(conf);

List list = Arrays.asList("a","b","c","d");

JavaRDD parallelize = sc.parallelize(list,3);//repartition

parallelize

.mapPartitionsWithIndex(new Function2, Iterator>() {

@Overridepublic Iterator call(Integer integer, Iterator stringIterator) throwsException {

List l = newArrayList();while(stringIterator.hasNext()){

String next=stringIterator.next();

System.out.println("分区id："+integer+"--值："+next);

l.add(next+integer);

}returnl.iterator();

}

}

,false)

.repartition(2)

.mapPartitionsWithIndex(new Function2, Iterator>() {

@Overridepublic Iterator call(Integer integer, Iterator stringIterator) throwsException {

List list = newArrayList();while(stringIterator.hasNext()){

String next=stringIterator.next();

System.out.println("重分区后，分区id:"+integer+"---值"+next);

list.add(next);

}returnlist.iterator();

}

}

,false)

.collect();//coalesce

parallelize

.mapPartitionsWithIndex(new Function2, Iterator>() {

@Overridepublic Iterator call(Integer integer, Iterator stringIterator) throwsException {

List l = newArrayList();while(stringIterator.hasNext()){

String next=stringIterator.next();

System.out.println("分区id："+integer+"--值："+next);

l.add(next+integer);

}returnl.iterator();

}

}

,false)

.coalesce(2,false)

.mapPartitionsWithIndex(new Function2, Iterator>() {

@Overridepublic Iterator call(Integer integer, Iterator stringIterator) throwsException {

List list = newArrayList();while(stringIterator.hasNext()){

String next=stringIterator.next();

System.out.println("重分区后，分区id:"+integer+"---值"+next);

list.add(next);

}returnlist.iterator();

}

}

,false)

.collect();

}

}