第1关:集合并行化创建RDD
package step1;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;
import java.util.*;
public class JStudent {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setMaster("local").setAppName("JStudent");
JavaSparkContext sc = new JavaSparkContext(conf);
List<Tuple2<String,Integer>> list = Arrays.asList(
new Tuple2("bj",88),new Tuple2("sh",67),new Tuple2("gz",92),
new Tuple2("bj",94),new Tuple2("sh",85),new Tuple2("gz",95),
new Tuple2("bj",72),new Tuple2("sh",69),new Tuple2("gz",98));
/********** Begin **********/
// 第一步:创建RDD
JavaPairRDD<String, Integer> rdd = sc.parallelizePairs(list);
// 第二步:把相同key的进行聚合
JavaPairRDD<String, Integer> totalScores = rdd.reduceByKey(Integer::sum);
// 第三步:输出
List<Tuple2<String, Integer>> results = totalScores.collect();
for (Tuple2<String, Integer> result : results) {
System.out.println(result);
}
/********** End **********/
sc.stop();
}
}
第2关:读取外部数据集创建RDD
package step2;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;
import java.util.Arrays;
import java.util.List;
public class JTeachers {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setMaster("local").setAppName("JTeachers");
JavaSparkContext sc = new JavaSparkContext(conf);
String dataFile = "file:///root/step2_files";
/********** Begin **********/
// 第一步:以外部文件方式创建RDD
JavaRDD<String> linesRDD = sc.textFile(dataFile);
// 第二步:将文件中每行的数据切分,得到自己想要的返回值
JavaPairRDD<String, Integer> teacherCountsRDD = linesRDD
.mapToPair(line -> {
String[] parts = line.split(",");
return new Tuple2<>(parts[1], 1);
});
// 第三步:将相同的key进行聚合
JavaPairRDD<String, Integer> aggregatedRDD = teacherCountsRDD.reduceByKey(Integer::sum);
// 第四步:将结果收集起来
List<Tuple2<String, Integer>> results = aggregatedRDD.collect();
// 第五步:输出
for (Tuple2<String, Integer> result : results) {
System.out.println(result);
}
/********** End **********/
sc.stop();
}
}