spark java eclipse_在eclipse上搭建spark的java开发环境

package test.spark;

import scala.Tuple2;

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.function.FlatMapFunction;

import org.apache.spark.api.java.function.Function2;

import org.apache.spark.api.java.function.PairFunction;

import org.apache.spark.sql.SparkSession;

import java.util.Arrays;

import java.util.Iterator;

import java.util.List;

import java.util.regex.Pattern;

public final class WordCount {

private static final Pattern SPACE = Pattern.compile(" ");

public static void main(String[] args) throws Exception {

if (args.length < 1) {

System.err.println("Usage: JavaWordCount ");

System.exit(1);

}

SparkSession spark = SparkSession

.builder()

.appName("JavaWordCount")

.getOrCreate();

JavaRDD lines = spark.read().textFile(args[0]).javaRDD();

JavaRDD words = lines.flatMap(new FlatMapFunction() {

@Override

public Iterator call(String s) {

return Arrays.asList(SPACE.split(s)).iterator();

}

});

JavaPairRDD ones = words.mapToPair(

new PairFunction() {

@Override

public Tuple2 call(String s) {

return new Tuple2<>(s, 1);

}

});

JavaPairRDD counts = ones.reduceByKey(

new Function2() {

@Override

public Integer call(Integer i1, Integer i2) {

return i1 + i2;

}

});

List> output = counts.collect();

for (Tuple2,?> tuple : output) {

System.out.println(tuple._1() + ": " + tuple._2());

}

spark.stop();

}

}

右键选择刚才建立的项目，选择 run as => run configurations , 选择arguments，

在program arguments 填入一个测试字数的文件，比如read.txt

在vm arguments参数设置中 添加程序运行的环境参数 比如 -Dspark.master=local -Xmx1g

然后 运行程序

分享到：

2016-12-05 09:52

浏览 490

评论