在集群运行spark代码记录程序

p150-156

package com.spark.firstApp

import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.SparkContext._
/**
 * Created by root on 6/4/15.
 */
object HelloSpark {
   def main(args: Array[String]) {
      if (args.length!=2){
        System.err.println("Usage: HelloSpark <Input> <Output>")
        System.exit(1)
      }
     val conf = new SparkConf().setAppName("HelloSpark")
     val sc = new SparkContext(conf)

     sc.textFile(args(0)).map(_.split("\t")).filter(_.length == 7).map(x => (x(1),1)).reduceByKey(_+_).
     map(x => (x._2,x._1)).sortByKey(false).map(x=>(x._2,x._1)).saveAsTextFile(args(1))
     sc.stop
  }

}

运行

/usr/spark/bin/spark-submit --class com.spark.firstApp.HelloSpark ./FirstSparkAppjar/FirstSparkApp.jar /usr/data/SogouQ.sample ./SougouResult