spark集群计算

圆周率计算

./run-example  SparkPi 10

./run-example --master spark://hadoop101:7077,hadoop102:7077,hadoop103:7077 SparkPi 10

wordcount计算

public class WordCount {
    public static void main(String[] args) {
        JavaSparkContext jsc = new JavaSparkContext(new SparkConf()
                .setAppName("WordCount"));
        jsc.textFile(args[0]).flatMap(s -> {
            String[] ss = s.split(" ");
            return Arrays.asList(ss).iterator();
        }).mapToPair(s -> new Tuple2<>(s, 1))
                .reduceByKey((v1,v2)->v1+v2)
                .saveAsTextFile(args[1]);
        jsc.stop();
    }
}

./spark-submit  --master spark://hadoop101:7077  --class org.example.spark.WordCount /itcast/spark.jar  file:///itcast/test.txt  file:///itcast/wc