wordCount scala

用scala写的wordCount

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.log4j.{Level, Logger}

object wordCount {
  Logger.getLogger("org").setLevel(Level.ERROR)
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("workCount")
    conf.set("spark.testing.memory", "2147480000")//后面的值大于512m即可
    val sc = new SparkContext(conf)
    val data = sc.textFile("d://dev/bigdata/wc.txt")
    data.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).collect().foreach(println)
  }
}