Spark RDD算子（八）mapPartitions， mapPartitionsWithIndex

mapPartitions

mapPartitions与map转换操作类似，只不过映射函数的输入参数由RDD中的每一个元素变成了RDD中每一个分区的迭代器
def mapPartitions[U](f: Iterator[Double] => Iterator[U],preservesPartitioning: Boolean)(implicit evidence$6: scala.reflect.ClassTag[U]): org.apache.spark.rdd.RDD[U]

• 传入的参数是一个迭代器
• perservesPartitioning指明mapPartitions是否保留父RDD的partitions的分区信息

scala版本

将每个分区内元素乘2

// 分为三个区
 val rdd1: RDD[Int] = sc.parallelize(List(1,2,3,4,5,6),3)
 // mapPartitions需要传递
 val mapPartitionsRDD: RDD[Int] = rdd1.mapPartitions((f: Iterator[Int]) => {
   println(f)
   f.map(_ * 2)
 })
 mapPartitionsRDD.collect().foreach(println)

java版本

JavaRDD<Integer> rdd1 = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6));
JavaRDD<Integer> mapPartitionsRDD = rdd1.mapPartitions(new FlatMapFunction<Iterator<Integer>, Integer>() {
    @Override
    public Iterator<Integer> call(Iterator<Integer> i) throws Exception {
        List<Integer> list = new ArrayList<>();
        while (i.hasNext()) {
            Integer x = i.next();
            list.add(x);
        }
        return list.iterator();
            }
    });
List<Integer> collect = mapPartitionsRDD.collect();
for (Integer i : collect) {
    System.out.println(i);
}

mapPartitionsWithIndex

mapPartitionWithIndex和mapPartitions功能类似，只是输入参数时多了一个分区的ID
def mapPartitionsWithIndex[U](f: (Int, Iterator[Double]) => Iterator[U],preservesPartitioning: Boolean)(implicit evidence$9: scala.reflect.ClassTag[U]): org.apache.spark.rdd.RDD[U]

scala版本

val rdd1: RDD[Int] = sc.parallelize(List(1,2,3,4,5,6),3)
val f =(x:Int,y:Iterator[Int])=>{
  println(x+"号分区")
  y.map(_*2)
}
val  mapPartitionsWithIndexRDD: RDD[Int] = rdd1.mapPartitionsWithIndex(f)
mapPartitionsWithIndexRDD.collect.foreach(println)

java版本

JavaRDD<Integer> rdd1 = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6),3);
Function2<Integer, Iterator<Integer>, Iterator<String>> function2 = new Function2<Integer, Iterator<Integer>, Iterator<String>>() {
    @Override
    public Iterator<String> call(Integer v1, Iterator<Integer> v2) throws Exception {
        System.out.println(v1 + "号分区");
        List<String> list = new ArrayList<>();
        while (v2.hasNext()) {
            String s = "hello" + v2.next();
            list.add(s);
        }
        return list.iterator();
    }
};
// 第二个参数用于指明是否保留父RDD的partitions的分区信息
JavaRDD<String> mapPartitionsWithIndexRDD = rdd1.mapPartitionsWithIndex(function2, false);
List<String> collect = mapPartitionsWithIndexRDD.collect();
for (String s : collect) {
    System.out.println(s);
}