Spark RDD算子(十)PairRDD的Action(动作)算子countByKey、collectAsMap
countBykey
def countByKey(): Map[K, Long]
- 统计出现相同key的次数
Scala版本
package nj.zb.sparkstu
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object countByKey {
def main(args: Array[String]): Unit = {
val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("countByKeyScala")
val sc: SparkContext = new SparkContext(conf)
val rdd1: RDD[(Int, Int)] = sc.parallelize(List((1,1),(2,1),(2,2),(3,1),(3,2),(3,3)))
val rdd2: collection.Map[Int, Long] = rdd1.countByKey()
rdd2.foreach(println)
}
}
结果展示:
Java版本
package nj.zb.sparkstu;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
public class countByKeyJava {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setMaster("local[*]").setAppName("countByKeyJava");
JavaSparkContext sc = new JavaSparkContext(conf);
JavaRDD<Tuple2<Integer, Integer>> rdd1 = sc.parallelize(Arrays.asList(
new Tuple2<Integer, Integer>(1, 1),
new Tuple2<Integer, Integer>(2, 1),
new Tuple2<Integer, Integer>(2, 2),
new Tuple2<Integer, Integer>(3, 1),
new Tuple2<Integer, Integer>(3, 2),
new Tuple2<Integer, Integer>(3, 3)));
JavaPairRDD<Integer, Integer> rdd2 = JavaPairRDD.fromJavaRDD(rdd1);
Map<Integer, Long> rdd3 = rdd2.countByKey();
Set<Integer> rdd4 = rdd3.keySet();
for (Integer a :
rdd4) {
System.out.println(a+"->"+rdd3.get(a));
}
}
}
结果展示:
colleactAsMap
- 将Pair类型(键值对类型)的RDD转换成map
Scala版本
package nj.zb.sparkstu
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object collectAsMap {
def main(args: Array[String]): Unit = {
val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("collectAsMap")
val sc: SparkContext = new SparkContext(conf)
val rdd1: RDD[(Int, Int)] = sc.parallelize(List((1,1),(2,1),(2,2),(3,1),(3,2),(3,3)))
val rdd2: collection.Map[Int, Int] = rdd1.collectAsMap()
rdd2.foreach(println)
}
}
结果展示:
Java版本
package nj.zb.sparkstu;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
public class CollectAsMapJava {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setMaster("local[*]").setAppName("CollectAsMapJava");
JavaSparkContext sc = new JavaSparkContext(conf);
JavaRDD<Tuple2<Integer, Integer>> rdd1 = sc.parallelize(Arrays.asList(
new Tuple2<>(1, 1),
new Tuple2<>(2, 1),
new Tuple2<>(2, 2),
new Tuple2<>(3, 1),
new Tuple2<>(3, 2),
new Tuple2<>(3, 3)));
JavaPairRDD<Integer, Integer> rdd2 = rdd1.mapToPair(new PairFunction<Tuple2<Integer, Integer>, Integer, Integer>() {
@Override
public Tuple2<Integer, Integer> call(Tuple2<Integer, Integer> integerIntegerTuple2) throws Exception {
return integerIntegerTuple2;
}
});
Map<Integer, Integer> rdd3 = rdd2.collectAsMap();
Set<Integer> rdd4 = rdd3.keySet();
for (Integer a :
rdd4) {
System.out.println(a+"->"+rdd3.get(a));
}
}
}
注意:这里不能直接用JavaPairRDD.fromJavaRDD()转换,不能会报错
结果展示: