一. Spark代码流程
- 创建SparkConf对象
可以设置Application name。
可以设置运行模式及资源需求。 - 创建SparkContext对象
- 基于Spark的上下文创建一个RDD,对RDD进行处理。
- 应用程序中要有Action类算子来触发Transformation类算子执行。
- 关闭Spark上下文对象SparkContext。
1. Scala代码
- 代码1
package com.hjf.core import org.apache.spark.rdd.RDD import org.apache.spark.{SparkConf, SparkContext} object WordCount { def main(args: Array[String]): Unit = { // 创建SparkConf对象 val conf: SparkConf = new SparkConf() conf.setMaster("local").setAppName("word count") // 创建SparkContext对象 val sc: SparkContext = new SparkContext(conf) // 指定日志等级 sc.setLogLevel("Error") // 按行读取文件中的内容 val line: RDD[String] = sc.textFile("./data/words.txt") // 按空格切割 val word: RDD[String] = line.flatMap(_.split(" ")) // 将单词转成(单词, 1)的格式 val map: RDD[(String, Int)] = word.map(one => new Tuple2(one, 1)) // 按key进行分组累加 val result: RDD[(String, Int)] = map.reduceByKey((v1, v2) => v1 + v2) // 打印 result.foreach(println(_)) // 简化版本: // sc.textFile("./data/words.txt").flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _).foreach(println(_)) // 关闭 sc.stop() } }
- 简化版本:
package com.hjf.core import org.apache.spark.rdd.RDD import org.apache.spark.{SparkConf, SparkContext} object WordCount { def main(args: Array[String]): Unit = { val conf: SparkConf = new SparkConf() conf.setMaster("local").setAppName("word count") val sc: SparkContext = new SparkContext(conf) // 指定日志等级 sc.setLogLevel("Error") sc.textFile("./data/words.txt").flatMap(_.split(" ")) .map((_, 1)).reduceByKey(_ + _).foreach(println(_)) // 关闭 sc.stop() } }
2. Java代码
-
Java7代码
package java7; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.api.java.function.VoidFunction; import scala.Tuple2; import java.util.Arrays; import java.util.Iterator; public class WordCount { public static void main(String[] args) { SparkConf conf = new SparkConf(); conf.setAppName("word count").setMaster("local"); JavaSparkContext sc = new JavaSparkContext(conf); sc.setLogLevel("Error"); JavaRDD<String> lines = sc.textFile("./data/words.txt"); JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() { @Override public Iterator<String> call(String s) throws Exception { String[] one = s.split(" "); return Arrays.asList(one).iterator(); } }); JavaPairRDD<String, Integer> pairRDD = words.mapToPair(new PairFunction<String, String, Integer>() { @Override public Tuple2<String, Integer> call(String s) throws Exception { return new Tuple2<String, Integer>(s, 1); } }); JavaPairRDD<String, Integer> result = pairRDD.reduceByKey(new Function2<Integer, Integer, Integer>() { @Override public Integer call(Integer v1, Integer v2) throws Exception { return v1 + v2; } }); result.foreach(new VoidFunction<Tuple2<String, Integer>>() { @Override public void call(Tuple2<String, Integer> ele) throws Exception { System.out.println(ele); } }); sc.stop(); } }
-
java8代码
package java8; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import scala.Tuple2; import java.util.Arrays; /** * @author Jiang锋时刻 * @create 2020-07-21 22:57 */ public class WordCount { public static void main(String[] args) { SparkConf conf = new SparkConf(); conf.setMaster("local").setAppName("word count"); JavaSparkContext sc = new JavaSparkContext(conf); sc.setLogLevel("Error"); JavaRDD<String> lines = sc.textFile("./data/words.txt"); JavaRDD<String> words = lines.flatMap(one -> Arrays.asList(one.split(" ")).iterator()); JavaPairRDD<String, Integer> pairRDD = words.mapToPair(one -> new Tuple2<>(one, 1)); JavaPairRDD<String, Integer> result = pairRDD.reduceByKey((v1, v2) -> v1 + v2); result.foreach(one -> System.out.println(one)); sc.stop(); } }