1. 在spark-shell 中实现wordcount
sc.textFile("hdfs://hadoop- 101:9000/a.txt").flatMap(_.split("")).map((_,1)).reduceByKey(_+_).collect
2. 在IDEA中实现
object WordCount extends App {
val conf: SparkConf = new SparkConf().setAppName("WordCount").setMaster("local")//本地调试
val properties = System.getProperties
properties.setProperty("HADOOP_USER_NAME", "root")
val sc = new SparkContext(conf)
val lines: RDD[String] = sc.textFile("hdfs://hadoop-101:9000/a.txt")
val words: RDD[String] = lines.flatMap(x => x.split(" "))
val wordAndOne: RDD[(String, Int)] = words.map(x => (x, 1))
val reduced: RDD[(String, Int)] = wordAndOne.reduceByKey(_ + _)
reduced.saveAsTextFile("hdfs://hadoop-101:9000/e")
sc.stop()
}