Spark使用小结：Java版的GroupByKey示例

Spark Java版的GroupByKey示例

感觉reduceByKey只能完成一些满足交换率，结合律的运算，如果想把某些数据聚合到一些做一些操作，得换groupbykey

比如下面：我想把相同key对应的value收集到一起，完成一些运算（例如拼接字符串，或者去重）

public class SparkSample {
    private static final Pattern SPACE = Pattern.compile(" ");

    public static void main(String args[]) {
        SparkConf sparkConf = new SparkConf();
        sparkConf.setAppName("Spark_GroupByKey_Sample");
        sparkConf.setMaster("local");

        JavaSparkContext context = new JavaSparkContext(sparkConf);

        List<Integer> data = Arrays.asList(1,1,2,2,1);
        JavaRDD<Integer> distData= context.parallelize(data);

        JavaPairRDD<Integer, Integer> firstRDD = distData.mapToPair(new PairFunction<Integer, Integer, Integer>() {
            @Override
            public Tuple2<Integer, Integer> call(Integer integer) throws Exception {
                return new Tuple2(integer, integer*integer);
            }
        });

        JavaPairRDD<Integer, Iterable<Integer>> secondRDD = firstRDD.groupByKey();

        List<Tuple2<Integer, String>> reslist = secondRDD.map(new Function<Tuple2<Integer, Iterable<Integer>>, Tuple2<Integer, String>>() {
            @Override
            public Tuple2<Integer, String> call(Tuple2<Integer, Iterable<Integer>> integerIterableTuple2) throws Exception {
                int key = integerIterableTuple2._1();
                StringBuffer sb = new StringBuffer();
                Iterable<Integer> iter = integerIterableTuple2._2();
                for (Integer integer : iter) {
                        sb.append(integer).append(" ");
                }
                return new Tuple2(key, sb.toString().trim());
            }
        }).collect();


        for(Tuple2<Integer, String> str : reslist) {
            System.out.println(str._1() + "\t" + str._2() );
        }
        context.stop();
    }
}