Spark Java Api Lambda 表达式 WordCount 例子

 注意：Lambda表达式Java8以后的版本才有

pom文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.smj</groupId>
    <artifactId>test</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>2.2.2</version>
        </dependency>

    </dependencies>
</project>

源码

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;

public class LambdaSparkWordCount {
    public static void main(String[] args) {
        // 1 创建编程入口类
        SparkConf conf = new SparkConf();
        conf.setMaster("local[*]");
        conf.setAppName(LambdaSparkWordCount.class.getSimpleName());

        JavaSparkContext jsc = new JavaSparkContext(conf);

        // 2 加载外部数据，形成spark中的计算的编程模型rdd
        JavaRDD<String> linesRDD = jsc.textFile("file:\\D:\\Workspace\\test\\src\\main\\resources\\hello.txt");

        // 3 对加载的数据进行各种业务逻辑操作，转换操作transformation
        System.out.println("------ 拆分rdd数据 ------");
        JavaRDD<String> wordsRDD = (JavaRDD<String>) linesRDD.flatMap(line -> Arrays.asList(line.split(",")).iterator());
        wordsRDD.foreach(t -> System.out.println(t));

        System.out.println("------ 拼装成键值对 ------");
        JavaPairRDD<String, Integer> pairsRDD = (JavaPairRDD<String, Integer>) wordsRDD.mapToPair(word -> new Tuple2<String, Integer>(word, 1));
        pairsRDD.foreach(t -> System.out.println(t._1 + "-" + t._2));

        System.out.println("------ 相同的key，累加value ------");
        JavaPairRDD<String, Integer> retRDD = pairsRDD.reduceByKey((v1, v2) -> v1 + v2);
        retRDD.foreach(t -> System.out.println(t._1 + "-" + t._2));
    }
}