1.配置profile
将spark解压到指定目录
tar -zxf spark-3.1.2.tgz -C /usr/local
vim /etc/profile
export SPARK_HOME=/usr/spark-3.1.2-bin-hadoop3.2
export PATH=$PATH:${SPARK_HOME}/bin
2.spark-env.sh配置
- 进入目录spark/conf
cd /usr/local/spark-3.1.2-bin-hadoop3.2/conf
- 复制spark-env.sh.template 名命为spark-env.sh进行配置
cp -i spark-env.sh.template spark-env.sh
- 进入spark-env.sh
vim spark-env.sh
- 末尾添加代码
export JAVA_HOME=/usr/java/jdk1.8.0_151
export HADOOP_CONF_DIR=/usr/local/spark-3.1.2-bin-hadoop3.2
export SPARK_MASTER_IP=master
export SPARK_MASTER_PORT=7077
export SPARK_WORKER_MEMORY=512m
export SPARK_WORKER_CORES=1
export SPARK_EXECUTOR_MEMORY=512m
export SPARK_EXECUTOR_CORES=1
export SPARK_WORKER_TNSTANCES=1
3.配置workers
-
复制workers.template 名命workers进行配置
cp -i workers.template workers
- vim 创建workers在本目录下
4.配置spark-defaults.conf
- 复制spark-defaults.conf.template名为spark-defaults.conf保存到当前目录
cp -i spark-defaults.conf.template spark-defaults.conf
- 进入spark-defaults.conf
vim spark-defaults.conf
:wqspark.master spark://master:7077
spark.eventLog.enabled true
spark.eventLog.dir hdfs://master:8020/spark-logs
spark.history.fs.loDirectory hdfs://master:8020/spark-logs
拷贝spark
scp -r /usr/local/spark-3.1.2-bin-hadoop3.2 slaver1:/usr/local
scp -r /usr/local/spark-3.1.2-bin-hadoop3.2/ slaver2:/usr/local
启动集群
- 先启动Hadoop
- 进入spark的sbin目录
cd usr/local/spark-3.1.2-bin-hadoop3.2/sbin
启动
./start-all.sh
动Hadoop
- 进入spark的sbin目录
cd usr/local/spark-3.1.2-bin-hadoop3.2/sbin
启动
./start-all.sh
总结:就是我最近搭建环境的详细步骤,仅供参考