运行JavaWordCount

运行hadoop:

这里写图片描述

JavaWordCount.java:
import scala.Tuple2;

import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.SparkSession;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;

public final class JavaWordCount {
  private static final Pattern SPACE = Pattern.compile(" ");

  public static void main(String[] args) throws Exception {

    //如果没有输入参数,则提示需要文件
    if (args.length < 1) {
      System.err.println("Usage: JavaWordCount <file>");
      System.exit(1);
    }
    //builder():Creates a SparkSession.Builder for constructing a SparkSession.
    //appName():Sets a name for the application, which will be shown in the Spark web UI. If no application name is set, a randomly generated name will be used.
    //getOrCreate():Gets an existing SparkSession or, if there is no existing one, creates a new one based on the options set in this builder.
    SparkSession spark = SparkSession
      .builder()
      .appName("JavaWordCount")
      .getOrCreate();
    //read():Returns a DataFrameReader that can be used to read non-streaming data in as a DataFrame.
    //textFile():Loads text files and returns a Dataset of String.
    //javaRDD():Returns the content of the Dataset as a JavaRDD of Ts.
    JavaRDD<String> lines = spark.read().textFile(args[0]).javaRDD();
    //public static <U> JavaRDD<U> flatMap(FlatMapFunction<T,U> f)
    JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
      @Override
      public Iterator<String> call(String s) {
          //将String分隔并且变成迭代器
        return Arrays.asList(SPACE.split(s)).iterator();
      }
    });
    //public static <K2,V2> JavaPairRDD<K2,V2> mapToPair(PairFunction<T,K2,V2> f)
    JavaPairRDD<String, Integer> ones = words.mapToPair(
      new PairFunction<String, String, Integer>() {
        @Override
        public Tuple2<String, Integer> call(String s) {
          return new Tuple2<String, Integer>(s, 1);
        }
      });
    //public JavaPairRDD<K,V> reduceByKey(Function2<V,V,V> func)
    JavaPairRDD<String, Integer> counts = ones.reduceByKey(
      new Function2<Integer, Integer, Integer>() {
        @Override
        public Integer call(Integer i1, Integer i2) {
          return i1 + i2;
        }
      });
    //public static java.util.List<T> collect()
    List<Tuple2<String, Integer>> output = counts.collect();
    for (Tuple2<?,?> tuple : output) {
      System.out.println(tuple._1() + ": " + tuple._2());
    }
    spark.stop();
  }
}
pom.xml:

这里写图片描述
这里要添加好依赖关系,否则编译不成功

find:

这里写图片描述
可以没有./spark-warehouse

编译:

mvn package启动编译,当出现BUILD SUCCESS时编译成功

启动运行:

这里写图片描述
file:///usr/local/spark/README.md是执行时输入的文件,这个可以修改,看代码可知

运行结果:

这里写图片描述

关闭hadoop:

这里写图片描述

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值