spark 的wordcount 俩种实现

1. 在spark-shell 中实现wordcount

   sc.textFile("hdfs://hadoop-   101:9000/a.txt").flatMap(_.split("")).map((_,1)).reduceByKey(_+_).collect 

2. 在IDEA中实现

object WordCount extends  App {
  val conf: SparkConf = new SparkConf().setAppName("WordCount").setMaster("local")//本地调试
  val properties = System.getProperties
  properties.setProperty("HADOOP_USER_NAME", "root")
  
  val sc = new SparkContext(conf)

  val lines: RDD[String] = sc.textFile("hdfs://hadoop-101:9000/a.txt")

  val words: RDD[String] = lines.flatMap(x => x.split(" "))


  val wordAndOne: RDD[(String, Int)] = words.map(x => (x, 1))
  val reduced: RDD[(String, Int)] = wordAndOne.reduceByKey(_ + _)

  reduced.saveAsTextFile("hdfs://hadoop-101:9000/e")


  sc.stop()

}