package demo
import org.apache.spark.{SparkConf, SparkContext}
//开发一个scala版本的wordcount
/*
bin/spark-submit --master spark://bigdata111:7077 --class demo.MyWordCount /root/temp/mywordcount.jar hdfs://192.168.157.111:9000/input/data.txt hdfs://192.168.157.111:9000/output/0608/wc2
*/
object MyWordCount {
def main(args: Array[String]): Unit = {
//配置
val conf = new SparkConf().setAppName("MyWordCount") //.setMaster("local")
//核心创建一个SparkContext
val sc = new SparkContext(conf)
//使用sc对象执行任务
// sc.textFile("hdfs://192.168.157.111:9000/input/data.txt")
// .flatMap(_.split(" "))
// .map((_,1))
// .reduceByKey(_+_)
// .saveAsTextFile("hdfs://192.168.157.111:9000/output/0608/wc1")
sc.textFile(args(0))
.flatMap(_.split(" "))
.map((_,1))
.reduceByKey(_+_)
.saveAsTextFile(args(1))
//停止SparkContext对象
sc.stop()
}
}
Spark编写wordcount程序
最新推荐文章于 2022-04-08 11:18:27 发布