spark-RDD集合操作

/**
 * Created by hbin on 2016/12/9.
 */
import java.util.Arrays;
import java.util.List;

import io.netty.util.internal.StringUtil;
import org.apache.commons.lang.StringUtils;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;


/**
 * spark对数据的核心抽象  RDD（弹性分布式数据集）
 * RDD就是分布式的元素集合,在spark中对数据的所有操作不外乎创建RDD
 * 转化已有RDD以及调用RDD操作进行求值,spark会自动将RDD中的数据分发到集群上，
 * 并将操作并行化
 */
public class BasicMap {
    public static void main(String[] args) throws Exception {
        SparkConf sparkConf = new SparkConf().setAppName("JavaSparkPi");
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        JavaRDD<String> RDD1=jsc.parallelize(Arrays.asList("A","B","C","D"),6);
        JavaRDD<String> RDD2=jsc.parallelize(Arrays.asList("E","F","G","H","A","B"),6);
        System.out.println("union :"+RDD1.union(RDD2).collect());
        System.out.println("intersection :"+RDD1.intersection(RDD2).collect());
        System.out.println("subtract :"+RDD1.subtract(RDD2).collect());
        System.out.println("cartesian :"+RDD1.cartesian(RDD2).collect());

//        JavaRDD<String> words=lines.flatMap(new FlatMapFunction<String, String>() {
//            @Override
//            public Iterable<String> call(String line) throws Exception {
//                return Arrays.asList(line.split("- "));
//            }
//        });
//        System.out.println("worls :"+words.collect());
    }
}
执行结果

union :[A, B, C, D, E, F, G, H, A, B]

intersection :[B, A]

subtract :[C, D]

cartesian :[(A,E), (A,F), (A,G), (A,H), (A,A), (A,B), (B,E), (B,F), (B,G), (B,H), (B,A), (B,B), (C,E), (C,F), (C,G), (C,H), (C,A), (C,B), (D,E), (D,F), (D,G), (D,H), (D,A), (D,B)]