将Spark部署到Hadoop YARN上

用的是Hadoop2.2.0

Spark0.9.1

新版本的spark中spark-class是在bin目录下

scala还是可以用scala2.9.3版本

下面是wordcount代码，参照董西城的博客

import org.apache.spark. _

import SparkContext. _

object WordCount {

   def main(args : Array[String]) {

     if (args.length ! = 3 ){

       println( "usage is org.test.WordCount <master> <input> <output>" )

       return

     }

     val sc = new SparkContext(args( 0 ), "WordCount" ,

     System.getenv( "SPARK_HOME" ), Seq(System.getenv( "SPARK_TEST_JAR" )))

     val textFile = sc.textFile(args( 1 ))

     val result = textFile.flatMap(line = > line.split( "\\s+" ))

         .map(word = > (word, 1 )).reduceByKey( _ + _ )

     result.saveAsTextFile(args( 2 ))

   }

}