运行JavaWordCount

最新推荐文章于 2023-07-07 17:50:29 发布

浮云若飞

最新推荐文章于 2023-07-07 17:50:29 发布

阅读量696

点赞数

分类专栏： java 文章标签： spark

本文链接：https://blog.csdn.net/qq_34911465/article/details/54948753

版权

java 专栏收录该内容

14 篇文章 0 订阅

订阅专栏

运行hadoop：

这里写图片描述

JavaWordCount.java:

import scala.Tuple2;

import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.SparkSession;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;

public final class JavaWordCount {
  private static final Pattern SPACE = Pattern.compile(" ");

  public static void main(String[] args) throws Exception {

    //如果没有输入参数，则提示需要文件
    if (args.length < 1) {
      System.err.println("Usage: JavaWordCount <file>");
      System.exit(1);
    }
    //builder():Creates a SparkSession.Builder for constructing a SparkSession.
    //appName():Sets a name for the application, which will be shown in the Spark web UI. If no application name is set, a randomly generated name will be used.
    //getOrCreate():Gets an existing SparkSession or, if there is no existing one, creates a new one based on the options set in this builder.
    SparkSession spark = SparkSession
      .builder()
      .appName("JavaWordCount")
      .getOrCreate();
    //read():Returns a DataFrameReader that can be used to read non-streaming data in as a DataFrame.
    //textFile():Loads text files and returns a Dataset of String.
    //javaRDD():Returns the content of the Dataset as a JavaRDD of Ts.
    JavaRDD<String> lines = spark.read().textFile(args[0]).javaRDD();
    //public static <U> JavaRDD<U> flatMap(FlatMapFunction<T,U> f)
    JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
      @Override
      public Iterator<String> call(String s) {
          //将String分隔并且变成迭代器
        return Arrays.asList(SPACE.split(s)).iterator();
      }
    });
    //public static <K2,V2> JavaPairRDD<K2,V2> mapToPair(PairFunction<T,K2,V2> f)
    JavaPairRDD<String, Integer> ones = words.mapToPair(
      new PairFunction<String, String, Integer>() {
        @Override
        public Tuple2<String, Integer> call(String s) {
          return new Tuple2<String, Integer>(s, 1);
        }
      });
    //public JavaPairRDD<K,V> reduceByKey(Function2<V,V,V> func)
    JavaPairRDD<String, Integer> counts = ones.reduceByKey(
      new Function2<Integer, Integer, Integer>() {
        @Override
        public Integer call(Integer i1, Integer i2) {
          return i1 + i2;
        }
      });
    //public static java.util.List<T> collect()
    List<Tuple2<String, Integer>> output = counts.collect();
    for (Tuple2<?,?> tuple : output) {
      System.out.println(tuple._1() + ": " + tuple._2());
    }
    spark.stop();
  }
}