用的是Hadoop2.2.0
Spark0.9.1
新版本的spark中spark-class是在bin目录下
scala还是可以用scala2.9.3版本
下面是wordcount代码,参照董西城的博客
import
org.apache.spark.
_
import
SparkContext.
_
object
WordCount {
def
main(args
:
Array[String]) {
if
(args.length !
=
3
){
println(
"usage is org.test.WordCount <master> <input> <output>"
)
return
}
val
sc
=
new
SparkContext(args(
0
),
"WordCount"
,
System.getenv(
"SPARK_HOME"
), Seq(System.getenv(
"SPARK_TEST_JAR"
)))
val
textFile
=
sc.textFile(args(
1
))
val
result
=
textFile.flatMap(line
=
> line.split(
"\\s+"
))
.map(word
=
> (word,
1
)).reduceByKey(
_
+
_
)
result.saveAsTextFile(args(
2
))
}
}