java8 wordcount_Spark2.x与Java8下WordCount示例

spark深度学习指南+进阶实战2册

123.9元

包邮

(需用券)

去购买 >

不用 lambda 的基础版

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.api.java.function.FlatMapFunction;

import org.apache.spark.api.java.function.Function2;

import org.apache.spark.api.java.function.PairFunction;

import org.apache.spark.sql.SparkSession;

import scala.Tuple2;

import java.io.Serializable;

import java.util.Arrays;

import java.util.Iterator;

public class WordCount implements Serializable {

public static void main(String[] args) {

// 输入文件

String wordFile = "/user/walker/input/wordcount/idea.txt";

SparkSession spark = SparkSession.builder()

.appName("wordcount")

.config("spark.executor.instances", 10)

.config("spark.executor.memory", "4g")

.config("spark.executor.cores", 1)

.config("spark.hadoop.mapreduce.output.fileoutputformat.compress", false)

.getOrCreate();

JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());

JavaRDD hdfstext = jsc.textFile(wordFile);

// 切分

JavaRDD words = hdfstext.flatMap(new FlatMapFunction() {

public Iterator call(String x) {

return Arrays.asList(x.split(" ")).iterator();

}

});

// 单次计 1

JavaPairRDD pairs = words.mapToPair(new PairFunction() {

public Tuple2 call(String word) {

return new Tuple2<>(word, 1);

}

});

// 累加 1

JavaPairRDD wordCounts = pairs.reduceByKey(new Function2() {

public Integer call(Integer v1, Integer v2) {

return v1 + v2;

}

}).repartition(1);

// 输出目录

String outDir = "/user/walker/output/wordcount";

wordCounts.saveAsTextFile(outDir);

jsc.close();

}

}

用 lambda 的基础版

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.sql.SparkSession;

import scala.Tuple2;

import java.io.Serializable;

import java.util.Arrays;

public class WordCount2 implements Serializable {

public static void main(String[] args) {

// 输入文件

String wordFile = "/user/walker/input/wordcount/idea.txt";

SparkSession spark = SparkSession.builder()

.appName("wordcount")

.config("spark.executor.instances", 10)

.config("spark.executor.memory", "4g")

.config("spark.executor.cores", 1)

.config("spark.hadoop.mapreduce.output.fileoutputformat.compress", false)

.getOrCreate();

JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());

JavaRDD hdfstext = jsc.textFile(wordFile);

// 切分

JavaRDD words = hdfstext.flatMap(line -> Arrays.asList(line.split(" ")).iterator());

// 单次计 1

JavaPairRDD pairs = words.mapToPair(word -> new Tuple2<>(word, 1));

// 累加 1

JavaPairRDD wordCounts = pairs.reduceByKey((v1, v2) -> v1 + v2).repartition(1);

// 输出目录

String outDir = "/user/walker/output/wordcount";

wordCounts.saveAsTextFile(outDir);

jsc.close();

}

}

用 lambda 的变形版

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.sql.SparkSession;

import scala.Tuple2;

import java.io.Serializable;

import java.util.Arrays;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

public class WordCount3 implements Serializable {

public static void main(String[] args) {

// 输入文件

String wordFile = "/user/walker/input/wordcount/idea.txt";

SparkSession spark = SparkSession.builder()

.appName("wordcount")

.config("spark.executor.instances", 10)

.config("spark.executor.memory", "4g")

.config("spark.executor.cores", 1)

.config("spark.hadoop.mapreduce.output.fileoutputformat.compress", false)

.getOrCreate();

JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());

JavaRDD hdfstext = jsc.textFile(wordFile);

// 切分

JavaRDD words = hdfstext.flatMap(line -> Arrays.asList(line.split(" ")).iterator());

// 计数

Map wordCounts = words.countByValue();

// 将 Map 转位 RDD

List> lst = new LinkedList<>();

wordCounts.forEach((k, v) -> lst.add(new Tuple2<>(k, v)));

JavaPairRDD result = jsc.parallelizePairs(lst).repartition(1);;

// 保存结果到 HDFS

String outDir = "/user/walker/output/wordcount"; //输出目录

result.saveAsTextFile(outDir);

jsc.close();

}

}

本文出自 walker snapshot

原文链接:https://segmentfault.com/a/1190000020488982

java 11官方入门(第8版)教材

79.84元

包邮

(需用券)

去购买 >