下载scala与spark并解压,添加至环境变量
编辑spark-env.sh 添加如下变量
export JAVA_HOME=/opt/apps/software/jdk1.8.0_201
export SCALA_HOME=/opt/apps/software/scala-2.11.8
export HADOOP_HOME=/opt/apps/software/hadoop-2.7.3
export HADOOP_CONF_DIR=/opt/apps/software/hadoop-2.7.3/etc/hadoop
#export SPARK_WORKER_MEMORY=500m
#export SPARK_WORKER_CORES=1
export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=hadoop1:2181,hadoop2:2181,hadoop3:2181 -Dspark.deploy.zookeeper.dir=/spark"
编辑slaves添加如下变量
hadoop1
hadoop2
hadoop3
进入spark下的sbin
./start-all.sh 启动所有
进入另外设备中启动master
./start-master.sh 即可高可用
spark测试:
spark-submit --master yarn --deploy-mode cluster --driver-memory 1000m --executor-memory 1000m --executor-cores 1 --class org.apache.spark.examples.SparkPi /opt/apps/software/spark-2.2.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.2.0.jar 10
spark-submit --master spark://hadoop1:7077 --deploy-mode cluster --driver-memory 1000m --executor-memory 1000m --executor-cores 1 --class org.apache.spark.examples.SparkPi /opt/apps/software/spark-2.2.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.2.0.jar 10