Windows下基于eclipse的Spark应用开发环境搭建

 转载自www.cnblogs.com/tovin/p/3822985.html

 上边的文章介绍的很好,根据自己的需要,做一些修改就可以了,下边是我的代码:

import java.util.List;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;

/**
 * 先参考http://www.cnblogs.com/tovin/p/3822985.html
 * 在spark的master上执行的命令如下:
 * [root@x02 bin]# spark-submit --class com.geo.SparkTest.App --master spark://x02:7077  /root/SparkTest-0.0.1-SNAPSHOT-jar-with-dependencies.jar /wjb/input/imanager.sql
 *
 */
public class App 
{
    public static void main( String[] args )
    {
    	String master="spark://x02:7077";
		SparkConf conf = new SparkConf().setAppName("topTest").setMaster(master);
    	JavaSparkContext sc = new JavaSparkContext(conf);
    	
    	System.out.println("------------------------------------------------------------");
    	for (String line : args) {
			System.out.println("arg:"+line);
		}
    	System.out.println("------------------------------------------------------------");
    	JavaRDD<String> lines = sc.textFile(args[0]);
    	List<String> tops = lines.top(2);
    	
    	for (String line : tops) {
			System.out.println("top:"+line);
		}
    }
}