spark单词统计+各个运行模式启动

object WordCount {
  //本地运行
    //System.setProperty("hadoop.home.dir","D:\\hadoop\\hadoop-2.9.2")
  def main(args: Array[String]): Unit = {
    //1.生成spark core总入口这个对象
    val conf = new SparkConf().setMaster("local").setAppName("wordcount")
    val sc = new SparkContext(conf)
    //2.单词统计
    sc.textFile(args(0))
      .flatMap(line => line.split(" "))
      .map(word => (word,1))
      .reduceByKey((x,y) => x+y)
      .saveAsTextFile(args(1))
        // .foreach(println)
    //3.关闭环境
    sc.stop()
  }
}

 

本地
bin/spark-submit  --class com.bw.spark.core.WordCount spark_demo-1.0-SNAPSHOT.jar file:/opt/spark-2.4.5/words.txt file:/opt/spark-2.4.5/output

standalone
bin/spark-submit --master spark://wml.com:7077  --class com.bw.spark.core.WordCount spark_demo-1.0-SNAPSHOT.jar file:/opt/spark-2.4.5/words.txt file:/opt/spark-2.4.5/output

yarn
bin/spark-submit --master yarn  --class com.bw.spark.core.WordCount spark_demo-1.0-SNAPSHOT.jar file:/opt/spark-2.4.5/words.txt file:/opt/spark-2.4.5/output