spark cogroup算子

 

java

 1 /** 
 2  *cogroup与join算子不同的是如果rdd中的一个key,对应多个value,则返回<Iterable<key>,Iterable<value>>
 3  *@author Tele
 4  */
 5 public class CogroupDemo {
 6     private static SparkConf conf = new SparkConf().setMaster("local").setAppName("congroupdemo");
 7     private static JavaSparkContext jsc = new JavaSparkContext(conf);
 8     public static void main(String[] args) {
 9         //每个学生有多门成绩
10         List<Tuple2<Integer,String>> studentList = Arrays.asList(
11                                                     new Tuple2<Integer,String>(1,"tele"), 
12                                                     new Tuple2<Integer,String>(1,"xx"), 
13                                                     new Tuple2<Integer,String>(2,"yeye"), 
14                                                     new Tuple2<Integer,String>(3,"wyc")
15                                                    );
16 
17         List<Tuple2<Integer,Integer>> scoreList = Arrays.asList(
18                                                   new Tuple2<Integer,Integer>(1,100),
19                                                   new Tuple2<Integer,Integer>(1,110),
20                                                   new Tuple2<Integer,Integer>(1,120),
21                                                   new Tuple2<Integer,Integer>(2,90),
22                                                   new Tuple2<Integer,Integer>(2,60),
23                                                   new Tuple2<Integer,Integer>(2,50),
24                                                   new Tuple2<Integer,Integer>(3,70),
25                                                   new Tuple2<Integer,Integer>(3,70)
26                                                   );
27         
28         JavaPairRDD<Integer, String> studentRDD = jsc.parallelizePairs(studentList);
29         JavaPairRDD<Integer, Integer> scoreRDD = jsc.parallelizePairs(scoreList);
30         
31         JavaPairRDD<Integer, Tuple2<Iterable<String>, Iterable<Integer>>> result = studentRDD.cogroup(scoreRDD);
32         result.foreach(new VoidFunction<Tuple2<Integer,Tuple2<Iterable<String>,Iterable<Integer>>>>() {
33             
34             private static final long serialVersionUID = 1L;
35 
36             @Override
37             public void call(Tuple2<Integer, Tuple2<Iterable<String>, Iterable<Integer>>> t) throws Exception {
38                 System.out.println("学号:" + t._1);
39                 System.out.println("姓名:" + t._2._1);
40                 System.out.println("成绩:" + t._2._2);
41                 
42             /*    System.out.print("成绩:[");
43                 t._2._2.forEach(i->System.out.print(i + ","));
44                 System.out.println("]");
45                 System.out.println("====================");*/
46                 
47             }
48         });
49         
50         jsc.close();
51     }
52 }

scala

 1 object CogroupDemo {
 2     def main(args: Array[String]): Unit = {
 3         val conf = new SparkConf().setMaster("local").setAppName("cogroupdemo");
 4         val sc = new SparkContext(conf);
 5         
 6         val studentArr = Array((1,"tele"),(2,"yeye"),(3,"wyc"));
 7         val scoreArr = Array((1,100),(1,200),(2,80),(2,300),(3,100));
 8         
 9         val studentRDD = sc.parallelize(studentArr,1);
10         val scoreRDD = sc.parallelize(scoreArr,1);
11         
12         val result = studentRDD.cogroup(scoreRDD);
13         result.foreach(t=>{
14           println("学号:" + t._1);
15           println("姓名:" + t._2._1.mkString(" "));
16           println("成绩:" + t._2._2.mkString(","));
17           println("============");
18         })
19     }
20 }

 

转载于:https://www.cnblogs.com/tele-share/p/10268576.html

发布了0 篇原创文章 · 获赞 5 · 访问量 4万+
展开阅读全文

没有更多推荐了,返回首页

©️2019 CSDN 皮肤主题: 编程工作室 设计师: CSDN官方博客

分享到微信朋友圈

×

扫一扫,手机浏览