java提交spark submit_spark-submit提交方式测试Demo

写一个小小的Demo测试一下Spark提交程序的流程

Maven的pom文件

1.7

1.7

UTF-8

1.6.1

org.apache.spark

spark-core_2.10

${spark.version}

redis.clients

jedis

2.7.1

org.apache.maven.plugins

maven-compiler-plugin

1.7

1.7

org.apache.maven.plugins

maven-shade-plugin

2.4.3

package

shade

*:*

META-INF/*.SF

META-INF/*.DSA

META-INF/*.RSA

编写一个蒙特卡罗求PI的代码

importjava.util.ArrayList;importjava.util.List;importorg.apache.spark.SparkConf;importorg.apache.spark.api.java.JavaRDD;importorg.apache.spark.api.java.JavaSparkContext;importorg.apache.spark.api.java.function.Function;importorg.apache.spark.api.java.function.Function2;importredis.clients.jedis.Jedis;/*** Computes an approximation to pi

* Usage: JavaSparkPi [slices]*/

public final classJavaSparkPi {public static void main(String[] args) throwsException {

SparkConf sparkConf= new SparkConf().setAppName("JavaSparkPi")/*.setMaster("local[2]")*/;

JavaSparkContext jsc= newJavaSparkContext(sparkConf);

Jedis jedis= new Jedis("192.168.49.151",19000);int slices = (args.length == 1) ? Integer.parseInt(args[0]) : 2;int n = 100000 *slices;

List l = new ArrayList(n);for (int i = 0; i < n; i++) {

l.add(i);