package spark;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.Optional;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;
import java.util.Arrays;
import java.util.List;
/**
* Created by wei on 2018/11/14.
*/
public class Join {
public static void main(String args[]) {
join();
}
public static void join() {
SparkConf conf = new SparkConf()
.setMaster("local")
.setAppName("join");
JavaSparkContext sc = new JavaSparkContext(conf);
List<Tuple2<String, String>> data = Arrays.asList(
new Tuple2<>("age", "s1"),
new Tuple2<>("name", "s2"),
new Tuple2<>("sex", "s3"),
new Tuple2<>("add", "s1")
);
JavaPairRDD<String, String> javaPairRDD = sc.parallelizePairs(data);
List<Tuple2<String, String>> data2 = Arrays.asList(
new Tuple2<>("age", "s1"),
new Tuple2<>("name", "s2"),
new Tuple2<>("sex", "s3"),
new Tuple2<>("age", "s134")
);
JavaPairRDD<String, String> javaPairRDD2 = sc.parallelizePairs(data2);
System.out.println("Join------------------------------>");
// (name,(s2,s2))
// (sex,(s3,s3))
// (age,(s1,s1))
// (age,(s1,s134))
javaPairRDD.join(javaPairRDD2).foreach(new VoidFunction<Tuple2<String, Tuple2<String, String>>>() {
@Override
public void call(Tuple2<String, Tuple2<String, String>> stringTuple2Tuple2) throws Exception {
System.out.println(stringTuple2Tuple2);
}
});
System.out.println("leftOuterJoin------------------------------>");
// (name,(s2,Optional[s2]))
// (sex,(s3,Optional[s3]))
// (add,(s1,Optional.empty))
// (age,(s1,Optional[s1]))
// (age,(s1,Optional[s134]))
javaPairRDD.leftOuterJoin(javaPairRDD2).foreach(new VoidFunction<Tuple2<String, Tuple2<String, Optional<String>>>>() {
@Override
public void call(Tuple2<String, Tuple2<String, Optional<String>>> stringTuple2Tuple2) throws Exception {
System.out.println(stringTuple2Tuple2);
}
});
System.out.println("rightOuterJoin------------------------------>");
// (name,(Optional[s2],s2))
// (sex,(Optional[s3],s3))
// (age,(Optional[s1],s1))
// (age,(Optional[s1],s134))
javaPairRDD.rightOuterJoin(javaPairRDD2).foreach(new VoidFunction<Tuple2<String, Tuple2<Optional<String>, String>>>() {
@Override
public void call(Tuple2<String, Tuple2<Optional<String>, String>> stringTuple2Tuple2) throws Exception {
System.out.println(stringTuple2Tuple2);
}
});
}
}