import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.api.java.function.VoidFunction; import scala.Tuple2; import java.util.Arrays; import java.util.Iterator; /** * Created by Knuth on 2018/1/30. */ public class SparkWordCount { public static void main(String[] args) { //1.创建一个SparkConf对象,本地spark模式 SparkConf conf = new SparkConf().setAppName("SparkWordCount").setMaster("local"); //2.创建JavaSparkContext对象,SparkContext是Spark所有功能的入口 JavaSparkContext jsc = new JavaSparkContext(conf); //3.创建一个初始的JavaRDD,读取文件的每一行 JavaRDD<String> wordLine = jsc.textFile("d:/wordcount.txt"); //4.split every wordLine JavaRDD<String> words = wordLine.flatMap(new FlatMapFunction<String, String>() { @Override public Iterator<String> call(String s) throws Exception { return Arrays.asList(s.split(" ")).iterator(); } }); //5.mapToPair JavaPairRDD<String,Integer> wordPair = words.mapToPair(new PairFunction<String, String,Integer>() { @Override public Tuple2<String,Integer> call(String word) throws Exception { return new Tuple2<String,Integer>(word,1); } }); //6.reduce JavaPairRDD<String,Integer> wordCount = wordPair.reduceByKey(new Function2<Integer, Integer, Integer>() { @Override public Integer call(Integer a, Integer b) throws Exception { return a+b; } }); //7.print wordCount.foreach(new VoidFunction<Tuple2<String, Integer>>() { @Override public void call(Tuple2<String, Integer> word) throws Exception { System.out.println(word._1+":"+word._2); } }); } }
spark wordcount Java代码实现
最新推荐文章于 2021-05-06 17:03:14 发布