spark简单测试

1 通过shell直接submit

2 通过Spark-shell执行

 

 

 for  i in {1..30};do spark-submit --class org.apache.spark.examples.JavaWordCount --master yarn --executor-memory 10g --driver-memory 20g --total-executor-cores 200 /opt/spark/examples/jars/spark-examples_2.11-2.4.4.jar  hdfs://master:8020/hello ;done

spark-shell  --master yarn --executor-memory 10g --driver-memory 20g --total-executor-cores 200 --jars /opt/spark/examples/jars/spark-examples_2.11-2.4.4.jar

import org.apache.spark.examples.JavaWordCount
var st= System.currentTimeMillis;
for (i <- 1 to 30){
    println(i)
    JavaWordCount.main(Array{"hdfs://master:8020/hello"})
    println(System.currentTimeMillis-st)
}