用aggregate计算RDD的平均值,代替map后面接fold的方式。package com.wellcomm.app;
import java.io.Serializable;
import java.util.Arrays;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;
public class AggTest {
public static void main(String[] args) {
JavaSparkContext sc = new JavaSparkContext(new SparkConf().setMaster("local").setAppName("hello"));
JavaRDD rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4));
AvgCount initial = new AvgCount(0, 0);
AvgCount result = rdd.aggregate(initial, addAndCount, combine);
System.out.println(result.avg());
}
static Function2 combine = new Function2() {
public AvgCount call(AvgCount a, AvgCount b) throws Exception {
// TODO Auto-generated method stub
a.total += b.total;
a.num += b.num;
return a;
}
};
static Function2 addAndCount = new Function2() {
public AvgCount call(AvgCount a, Integer x) throws Exception {
a.total += x;
a.num += 1;
return a;
}
};
}
class AvgCount implements Serializable {
private static final long serialVersionUID = 1L;
public AvgCount(int total, int num) {
this.total = total;
this.num = num;
}
public int total;
public int num;
public double avg() {
return total / (double) num;
}
}
推荐您阅读更多有关于“Spark,Java,”的文章