spark install

 

.bashrc

export HADOOP_HOME=/usr/local/hadoop/hadoop-2.6.4
export HADOOP_CONF_DIR=/usr/local/hadoop/hadoop-2.6.4/etc/hadoop
export SCALA_HOME=/usr/local/scala/scala-2.10.6
export SPARK_HOME=/usr/local/spark/spark-2.0.0-bin-hadoop2.6
export JAVA_HOME=/usr/java/jdk1.8.0
export JRE_HOME=${JAVA_HOME}/jre
export CLASS_PATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${SCALA_HOME}/bin:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH

 

 

spark_env.sh

 

export JAVA_HOME=/usr/java/jdk1.8.0

export SCALA_HOME=/usr/local/scala/scala-2.10.6

export HADOOP_HOME=/usr/local/hadoop/hadoop-2.6.4

export HADOOP_CONF_DIR=/usr/local/hadoop/hadoop-2.6.4/etc/hadoop

export SPARK_MASTER_IP=Master

export SPARK_WORKER_MEMORY=1g

export SPARK_EXECUTOR_MEMORY=1g

export SPARK_DRIVER_MEMORY=1G

export SPARK_WORKER_CORES=8

 

slaves

         Worker1

         Worker2

 

 

 

 

export SCALA_HOME=/usr/local/scala/scala-2.10.6

export SPARK_HOME=/usr/local/spark/spark-2.0.0-bin-hadoop2.6

 

 

spark-defaults.conf

 

spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"

spark.eventLog.enabled           true

spark.eventLog.dir               hdfs://Master:9000/historyserverforSpark

spark.yarn.historyServer.address Master:18080

spark.history.fs.logDirectory hdfs://Master:9000/historyserverforSpark

 

 

hadoop dfs -r 

hadoop dfs -mkdir /historyserverforSpark

 

spark-submit --class org.apache.spark.examples.SparkPi --master spark://Master:7077 ../examples/jars/spark-examples_2.11-2.0.0.jar    1000

  --class org.apache.spark.examples.SparkPi \

  --master spark://Master:7077 \

  ../examples/jars/spark-examples_2.11-2.0.0.jar    1000

转载于:https://www.cnblogs.com/kyostone/p/5855133.html