执行结果/** * Created by hbin on 2016/12/9. */ import java.util.Arrays; import java.util.List; import io.netty.util.internal.StringUtil; import org.apache.commons.lang.StringUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function; /** * spark对数据的核心抽象 RDD(弹性分布式数据集) * RDD就是分布式的元素集合,在spark中对数据的所有操作不外乎创建RDD * 转化已有RDD以及调用RDD操作进行求值,spark会自动将RDD中的数据分发到集群上, * 并将操作并行化 */ public class BasicMap { public static void main(String[] args) throws Exception { SparkConf sparkConf = new SparkConf().setAppName("JavaSparkPi"); JavaSparkContext jsc = new JavaSparkContext(sparkConf); JavaRDD<String> RDD1=jsc.parallelize(Arrays.asList("A","B","C","D"),6); JavaRDD<String> RDD2=jsc.parallelize(Arrays.asList("E","F","G","H","A","B"),6); System.out.println("union :"+RDD1.union(RDD2).collect()); System.out.println("intersection :"+RDD1.intersection(RDD2).collect()); System.out.println("subtract :"+RDD1.subtract(RDD2).collect()); System.out.println("cartesian :"+RDD1.cartesian(RDD2).collect()); // JavaRDD<String> words=lines.flatMap(new FlatMapFunction<String, String>() { // @Override // public Iterable<String> call(String line) throws Exception { // return Arrays.asList(line.split("- ")); // } // }); // System.out.println("worls :"+words.collect()); } }
union :[A, B, C, D, E, F, G, H, A, B]
intersection :[B, A]
subtract :[C, D]
cartesian :[(A,E), (A,F), (A,G), (A,H), (A,A), (A,B), (B,E), (B,F), (B,G), (B,H), (B,A), (B,B), (C,E), (C,F), (C,G), (C,H), (C,A), (C,B), (D,E), (D,F), (D,G), (D,H), (D,A), (D,B)]