spark api java_spark java api 开发

import scala.Tuple2;

import spark.api.java.JavaPairRDD;

import spark.api.java.JavaRDD;

import spark.api.java.JavaSparkContext;

import spark.api.java.function.FlatMapFunction;

import spark.api.java.function.Function2;

import spark.api.java.function.PairFunction;

import java.util.Arrays;

import java.util.List;

public class JavaWordCount {

public static void main(String[] args) throws Exception {

if (args.length 2) {

System.err.println("Usage: JavaWordCount ");

System.exit(1);

}

JavaSparkContext ctx = new JavaSparkContext(args[0], "JavaWordCount",

System.getenv("SPARK_HOME"), System.getenv("SPARK_EXAMPLES_JAR"));

JavaRDD lines = ctx.textFile(args[1], 1);

JavaRDD words = lines.flatMap(new FlatMapFunction() {

public Iterable call(String s) {

return Arrays.asList(s.split(" "));

}

});

JavaPairRDD ones = words.map(new PairFunction() {

public Tuple2 call(String s) {

return new Tuple2(s, 1);

}

});

JavaPairRDD counts = ones.reduceByKey(new Function2() {

public Integer call(Integer i1, Integer i2) {

return i1 + i2;

}

});

List> output = counts.collect();

for (Tuple2 tuple : output) {

System.out.println(tuple._1 + ": " + tuple._2);

}

System.exit(0);

}

}

运行： ./run spark/examples/JavaWordCount  local input.txt

local：不解析，自己查

Html代码

Hello World Bye World goole

运行的结果和haddoop中运行的JavaWordCount  一样

Html代码

goole: 1

World: 2

Hello: 1

Bye: 1

所有博客已经转移至leanote：http://blog.leanote.com/shiwei/

或者：http://luoshiwei.me/

分享到：

2015-04-22 14:10

浏览 1803

评论