安装spark 6657

解压安装spark
tar -zxvf spark-3.1.1-bin-hadoop3.2.tgz
改名
mv spark-3.1.1-bin-hadoop3.2/ spark

配置环境变量
cd /etc
vim profile

export SPARK_HOME=/opt/soft/spark
export PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH

使环境变量生效
source /etc/profile/

修改配置文件
(1)
cp workers.template workers
cp spark-env.sh.template spark-env.sh
cp spark-defaults.conf.template spark-defaults.conf

vim workers
master
slave01
slave02

vim spark-env.sh
  
JAVA_HOME=/opt/soft/jdk           
HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
SPARK_MASTER_HOST=master
SPARK_MASTER_PORT=7077
SPARK_MASTER_WEBUI_PORT=8080
SPARK_WORKER_CORES=2
SPARK_WORKER_MEMORY=3g
SPARK_WORKER_WEBUI_PORT=8081
SPARK_WORKER_DIR=/data/spark/work
SPARK_PID_DIR=/var/run

(2)

分发
1.配置文件
scp /etc/profile slave01:/etc/profile
scp /etc/profile slave02:/etc/profile

2.spark
scp -r /opt/soft/spark/ slave01:/opt/soft/spark/
scp -r /opt/soft/spark/ slave02:/opt/soft/spark/

启动Hadoop集群
start-all.sh

启动spark集群
cd /opt/soft/spark
./sbin/start-all.sh

进入spark shell模式(查看版本号）
spark-shell

访问webUI查看集群状态
master:8080