:代码
package com.test.mllib.test
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
object WorkCountApp {
def main(args: Array[String]) {
var filename = ""
args match {
case Array(f) => {
filename = f
}
case _ => {
System.err.println("Usage: WorkCountApp <filename>")
System.exit(1)
}
}
val sparkConf = new SparkConf().setAppName("WorkCountApp")
val sc = new SparkContext(sparkConf)
val textFile = sc.textFile(filename)
val wordCounts = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
wordCounts.foreach(println)
}
}
:本地运行
E:\DM\Spark\spark-1.5.2-bin-hadoop2.6\bin\spark-submit --master local[1] --class com.test.mllib.test.WorkCountApp E:\DM\code\projects\ch11-testit\target\ch11-testit-1.0.0.jar E:\DM\Spark\spark-1.5.2-bin-hadoop2.6\README.md
:Yarn运行
bin\spark-submit --class com.test.mllib.test.WorkCountApp --master yarn --deploy-mode client --executor-memory 256M --num-executors 1 E:\DM\code\projects\ch11-testit\target\ch11-testit-1.0.0.jar hdfs://localhost:19000/README.txt