体会一下:mapToPair()、countByKey()、reduceByKey()。
import org.apache.spark.Partitioner;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import scala.Tuple2;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
public class ReduceByKeyDemo {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("Simple Application").setMaster("local");
// 这一行非常耗时,目测耗时约10秒
JavaSparkContext sc = new JavaSparkContext(conf);
List<Integer> data = Arrays.asList(1, 2, 2, 3, 3, 3, 4, 5, 6, 7);
JavaRDD<Integer> javaRDD = sc.parallelize(data);
System.out.println("### List<Integer>转JavaRDD=" + javaRDD.collect());
//转化为K,V格式
JavaPairRDD<Integer,Integer> javaPairRDD = javaRDD.mapToPair(new PairFunction<Integer, Integer, Integer>() {
@Override
public Tuple2<Integer, Integer> call(Integer integer) {
return new Tuple2<Integer, Integer>(integer, 1);
}
});
System.out.println("### javaRDD.mapToPair(...)=" + javaPairRDD.collect());
Map<Integer, Long> countByKeyResultMap = javaPairRDD.countByKey();
System.out.println("### javaRDD.countByKey()=" + countByKeyResultMap);
JavaPairRDD<Integer,Integer> reduceByKeyRDD1 = javaPairRDD.reduceByKey(new Function2<Integer, Integer, Integer>() {
@Override
public Integer call(Integer v1, Integer v2) {
return v1 + v2;
}
});
System.out.println("### javaPairRDD.reduceByKey,方式1=" + reduceByKeyRDD1.collect());
//指定numPartitions
JavaPairRDD<Integer,Integer> reduceByKeyRDD2 = javaPairRDD.reduceByKey(new Function2<Integer, Integer, Integer>() {
@Override
public Integer call(Integer v1, Integer v2) {
return v1 + v2;
}
},2);
System.out.println("### javaPairRDD.reduceByKey,方式2=" + reduceByKeyRDD2.collect());
//自定义partition
JavaPairRDD<Integer,Integer> reduceByKeyRDD3 = javaPairRDD.reduceByKey(new Partitioner() {
@Override
public int numPartitions() {
return 2;
}
@Override
public int getPartition(Object o) {
return (o.toString()).hashCode() % numPartitions();
}
}, new Function2<Integer, Integer, Integer>() {
@Override
public Integer call(Integer v1, Integer v2) {
return v1 + v2;
}
});
System.out.println("### javaPairRDD.reduceByKey,方式3=" + reduceByKeyRDD3.collect());
}
}
体会一下:groupByKey
test.txt内容:
2010-05-04 11:50,10,10,10
2010-05-04 12:50,10,10,10
2010-05-04 12:50,10,10,10
2010-05-05 13:50,20,20,20
2010-05-05 13:50,20,20,20
2010-05-06 14:50,30,30,30
2010-05-06 14:50,30,30,30
在Spark-shell里执行:
import scala.io.Source
val source = Source.fromFile("D:/test.txt").getLines.toArray
val sourceRDD = sc.parallelize(source)
sourceRDD.map {
line =>
val lines = line.split(",")
(s"${lines(0)}", s"${lines(1)},${lines(2)},${lines(3)}")
}.groupByKey.map {
case (k, v) =>
var a, b, c = 0
v.foreach {
x =>
val r = x.split(",")
a += r(0).toInt
b += r(1).toInt
c += r(2).toInt
}
s"$k,$a,$b,$c"
}.foreach(println)
输出:
2010-05-04 11:50,10,10,10
2010-05-04 12:50,20,20,20
2010-05-05 13:50,40,40,40
2010-05-06 14:50,60,60,60