package com.pyrrha.examples;
import java.util.ArrayList;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.Optional;
import scala.Tuple2;
public class JoinTransformation {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("TransformationTest").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(conf);
join(sc);
sc.close();
}
/**
* @return:
ListA:
[(name,Jos), (name,Marry), (name1,Tom)]
ListB
[(name,Jos), (name,Black), (name2,David)]
-----------------join-------------------
[(name,(Jos,Jos)), (name,(Jos,Black)), (name,(Marry,Jos)), (name,(Marry,Black))]
-----------------leftOuterJoin-------------------
[(name,(Jos,Optional[Jos])), (name,(Jos,Optional[Black])), (name,(Marry,Optional[Jos])), (name,(Marry,Optional[Black])), (name1,(Tom,Optional.empty))]
-----------------rightOuterJoin-------------------
[(name,(Optional[Jos],Jos)), (name,(Optional[Jos],Black)), (name,(Optional[Marry],Jos)), (name,(Optional[Marry],Black)), (name2,(Optional.empty,David))]
-----------------fullOuterJoin-------------------
[(name,(Optional[Jos],Optional[Jos])), (name,(Optional[Jos],Optional[Black])), (name,(Optional[Marry],Optional[Jos])), (name,(Optional[Marry],Optional[Black])), (name2,(Optional.empty,Optional[David])), (name1,(Optional[Tom],Optional.empty))]
* @param sc
*/
public static void join(JavaSparkContext sc) {
List<Tuple2<String, String>> listA = new ArrayList<Tuple2<String, String>>();
listA.add(new Tuple2<String, String>("name","Jos"));
listA.add(new Tuple2<String, String>("name","Marry"));
listA.add(new Tuple2<String, String>("name1","Tom"));
List<Tuple2<String, String>> listB = new ArrayList<Tuple2<String, String>>();
listB.add(new Tuple2<String, String>("name","Jos"));
listB.add(new Tuple2<String, String>("name","Black"));
listB.add(new Tuple2<String, String>("name2","David"));
JavaPairRDD<String, String> RDDA = sc.parallelizePairs(listA);
JavaPairRDD<String, String> RDDB = sc.parallelizePairs(listB);
System.out.println("-----------------data-------------------");
System.out.println("ListA:");
System.out.println(RDDA.collect());
System.out.println("\n");
System.out.println("ListB");
System.out.println(RDDB.collect());
System.out.println("\n");
JavaPairRDD<String, Tuple2<String, String>> RDDC = RDDA.join(RDDB);
System.out.println("-----------------join-------------------");
System.out.println(RDDC.collect());
System.out.println("\n");
JavaPairRDD<String, Tuple2<String, Optional<String>>> RDDD = RDDA.leftOuterJoin(RDDB);
System.out.println("-----------------leftOuterJoin-------------------");
System.out.println(RDDD.collect());
System.out.println("\n");
JavaPairRDD<String, Tuple2<Optional<String>, String>> RDDE = RDDA.rightOuterJoin(RDDB);
System.out.println("-----------------rightOuterJoin-------------------");
System.out.println(RDDE.collect());
System.out.println("\n");
JavaPairRDD<String, Tuple2<Optional<String>, Optional<String>>> RDDF = RDDA.fullOuterJoin(RDDB);
System.out.println("-----------------fullOuterJoin-------------------");
System.out.println(RDDF.collect());
System.out.println("\n");
}
}