用scala写的wordCount
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.log4j.{Level, Logger}
object wordCount {
Logger.getLogger("org").setLevel(Level.ERROR)
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local").setAppName("workCount")
conf.set("spark.testing.memory", "2147480000")//后面的值大于512m即可
val sc = new SparkContext(conf)
val data = sc.textFile("d://dev/bigdata/wc.txt")
data.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).collect().foreach(println)
}
}