package cn.hhb.spark.core;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.broadcast.Broadcast;
import java.util.Arrays;
import java.util.List;
/**
* Created by dell on 2017/7/13.
*/
public class BroadcastVariable {
public static void main(String[] args) {
SparkConf conf = new SparkConf()
.setAppName("BroadcastVariable")
.setMaster("local")
.set("spark.testing.memory", "2147480000");
JavaSparkContext sc = new JavaSparkContext(conf);
/**
* 在java中,创建共享变量,就是调用SparkContext的broadcast方法
* 获取的返回结果Broadcast<Integer>类型
*/
final int factor = 3;
final Broadcast<Integer> factorBroadcast = sc.broadcast(factor);
List<Integer> numberList = Arrays.asList(1,2,3,4,5);
JavaRDD<Integer> number = sc.parallelize(numberList);
// 让集合中的每个数字,都乘以外部定义的那个factor
JavaRDD<Integer> multipleNumbers = number.map(new Function<Integer, Integer>() {
@Override
public Integer call(Integer number) throws Exception {
// 使用共享变量时,调用其value(),即可获取其内部共享的值
int factor = factorBroadcast.value();
return number * factor;
}
});
multipleNumbers.foreach(new VoidFunction<Integer>() {
@Override
public void call(Integer integer) throws Exception {
System.out.println(integer);
}
});
sc.close();
}
}
转载于:https://my.oschina.net/hehongbo/blog/1486268