package wordcount.WordCount
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
/**
* Created by Ryan on 2017/3/7.
*/
object scala {
def main(args: Array[String]) : Unit = {
val conf = new SparkConf()
conf.setAppName("WordCountApplication")
conf.setMaster("local")
val sc = new SparkContext(conf)
val lines = sc.textFile("/usr/softwares/spark/ReadMe.md", 3)
val words = lines.flatMap{line => line.split(" ")}
val pairs = words.map(words => (words, 1))
val wordCounts = pairs.reduceByKey(_+_)
wordCounts.foreach(wordNumberPair => println(wordNumberPair._1 + " : " + wordNumberPair._2))
sc.stop()
}
}
Spark学习笔记 --- scala实现Spark wordcount例子
最新推荐文章于 2022-08-05 15:27:50 发布
本文档记录了使用Scala编写的Spark WordCount程序。通过创建SparkConf,设置应用名和Master,读取文本文件,分割单词,进行reduceByKey操作来统计每个单词出现的次数,并打印结果。
摘要由CSDN通过智能技术生成