环境变量
/etc/profile
export JAVA_HOME=/root/apps/jdk1.8.0_60
export HADOOP_HOME=/root/apps/hadoop-2.8.0
export SCALA_HOME=/root/apps/scala-2.12.4
export SPARK_HOME=/root/apps/spark-2.11
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$SCALA_HOME
/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin
1、conf 目录下spark-env.sh 如下配置
export SCALA_HOME=/root/apps/scala-2.12.4/
export HADOOP_HOME=/root/apps/hadoop-2.8.0/
export HADOOP_CONF_DIR=/root/apps/hadoop-2.8.0/etc/hadoop/
export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER
-Dspark.deploy.zookeeper.url=hdp-01:2181,hdp-02:2181,hdp-03:2181
-Dspark.deploy.zookeeper.dir=/spark"
export SPARK_WORKER_MEMORY=1g
export SPARK_EXECUTOR_MEMORY=1g
export SPARK_DRIVER_MEMORY=1g
export SPARK_WORKER_CORES=4
2、conf 目录下spark-env.sh 如下配置
hdp-01
hdp-02
hdp-03
hdp-04