Point 1:AccumulatorValue
package com.spark.sharevariables.java;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.Accumulator;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction;
public class AccumulatorValue {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("AccumulatorValue").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(conf);
final Accumulator<Integer> sum = sc.accumulator(0, "Our Accumulator");
List<Integer> list = Arrays.asList(1,2,3,4,5);
JavaRDD<Integer> listRDD = sc.parallelize(list);
listRDD.foreach(new VoidFunction<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public void call(Integer num) throws Exception {
sum.add(num);
// System.out.println(sum.value());
}
});
System.out.println(sum.value());
sc.close();
}
}
Point 2:BroadCastValue
package com.spark.sharevariables.java;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.broadcast.Broadcast;
public class BroadCastValue {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("BroadCastValue").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(conf);
final int f = 3;
final Broadcast<Integer> broadCastFactor = sc.broadcast(f);
List<Integer> list = Arrays.asList(1,2,3,4,5);
JavaRDD<Integer> listRDD = sc.parallelize(list);
JavaRDD<Integer> results = listRDD.map(new Function<Integer, Integer>() {
private static final long serialVersionUID = 1L;
@Override
public Integer call(Integer num) throws Exception {
// return num * f;
// 只读的
return num * broadCastFactor.value();
}
});
results.foreach(new VoidFunction<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public void call(Integer num) throws Exception {
System.out.println(num);
}
});
sc.close();
}
}