生产环境实战spark （9）分布式集群 5台设备 SPARK集群安装

最新推荐文章于 2024-07-13 09:47:47 发布

段智华

最新推荐文章于 2024-07-13 09:47:47 发布

阅读量1.3k

点赞数

分类专栏：生产实战spark

热烈祝贺Gavin大咖2024年北京航空航天大学两本新书《Transformer&ChatGPT解密：原理、源码及案例》、《Transformer& Rasa 解密: 原理、源码及案例》出版发行

本文链接：https://blog.csdn.net/duan_zhihua/article/details/70739874

版权

生产实战spark 专栏收录该内容

13 篇文章 2 订阅

订阅专栏

生产环境实战spark （9）分布式集群 5台设备 SPARK集群安装

1, 上传spark到master，检查

[root@master rhzf_spark_setupTools]# ls
hadoop-2.6.5.tar.gz  jdk-8u121-linux-x64.tar.gz  scala-2.11.8.zip  spark-2.1.0-bin-hadoop2.6.tgz
[root@master rhzf_spark_setupTools]#

2，解压缩spark安装

[root@master rhzf_spark_setupTools]# tar -zxvf spark-2.1.0-bin-hadoop2.6.tgz

[root@master rhzf_spark_setupTools]# ls
hadoop-2.6.5.tar.gz  jdk-8u121-linux-x64.tar.gz  scala-2.11.8.zip  spark-2.1.0-bin-hadoop2.6  spark-2.1.0-bin-hadoop2.6.tgz
[root@master rhzf_spark_setupTools]# mv spark-2.1.0-bin-hadoop2.6 /usr/local
[root@master rhzf_spark_setupTools]# cd /usr/local
[root@master local]# ls
bin  games         include       lib    libexec             rhzf_spark_setupTools  scala-2.11.8  spark-2.1.0-bin-hadoop2.6
etc  hadoop-2.6.5  jdk1.8.0_121  lib64  rhzf_setup_scripts  sbin                   share         src
[root@master local]#

3，编辑 /etc/profile profile文件

export JAVA_HOME=/usr/local/jdk1.8.0_121
export SCALA_HOME=/usr/local/scala-2.11.8
export HADOOP_HOME=/usr/local/hadoop-2.6.5
export SPARK_HOME=/usr/local/spark-2.1.0-bin-hadoop2.6

export PATH=.:$PATH:$JAVA_HOME/bin:$SCALA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin

刷新生效

[root@master spark-2.1.0-bin-hadoop2.6]# source  /etc/profile

4，spark配置文件修改

[root@master spark-2.1.0-bin-hadoop2.6]# cd ..
[root@master local]# ls
bin  games         include       lib    libexec             rhzf_spark_setupTools  scala-2.11.8  spark-2.1.0-bin-hadoop2.6
etc  hadoop-2.6.5  jdk1.8.0_121  lib64  rhzf_setup_scripts  sbin                   share         src
[root@master local]# cd spark-2.1.0-bin-hadoop2.6
[root@master spark-2.1.0-bin-hadoop2.6]# ls
bin  conf  data  examples  jars  LICENSE  licenses  NOTICE  python  R  README.md  RELEASE  sbin  yarn
[root@master spark-2.1.0-bin-hadoop2.6]# cd conf
[root@master conf]# ls
docker.properties.template  log4j.properties.template    slaves.template               spark-env.sh.template
fairscheduler.xml.template  metrics.properties.template  spark-defaults.conf.template
[root@master conf]# mv spark-env.sh.template spark-env.sh
[root@master conf]# ls
docker.properties.template  log4j.properties.template    slaves.template               spark-env.sh
fairscheduler.xml.template  metrics.properties.template  spark-defaults.conf.template
[root@master conf]# vi spark-env.sh


export JAVA_HOME=/usr/local/jdk1.8.0_121
export SCALA_HOME=/usr/local/scala-2.11.8

export SPARK_MASTER_IP=10. 0.237
export SPARK_WORKER_MEMORY=2g
export HADOOP_CONF_DIR=/usr/local/hadoop-2.6.5/etc/hadoop
"spark-env.sh" 82L, 4180C written
[root@master conf]#

[root@master conf]# ls
docker.properties.template  log4j.properties.template    slaves.template               spark-env.sh
fairscheduler.xml.template  metrics.properties.template  spark-defaults.conf.template
[root@master conf]# mv  slaves.template  slaves
[root@master conf]# ls
docker.properties.template  fairscheduler.xml.template  log4j.properties.template  metrics.properties.template  slaves  spark-defaults.conf.template  spark-env.sh
[root@master conf]# vi  slaves

worker01
worker02
worker03
worker04

5，woker节点脚本分发

[root@master rhzf_setup_scripts]# ls
rhzf_hadoop.sh  rhzf_hosts_scp.sh  rhzf_jdk.sh  rhzf_scala.sh  rhzf_ssh.sh
[root@master rhzf_setup_scripts]# vi rhzf_spark.sh

#!/bin/sh
for i in  238 239 240 241
do
scp   -rq /usr/local/spark-2.1.0-bin-hadoop2.6  root@10 .$i:/usr/local/spark-2.1.0-bin-hadoop2.6
scp   -rq /etc/profile  root@10 .$i:/etc/profile
ssh   root@10. 0.$i source /etc/profile
done

[root@master rhzf_setup_scripts]# ls
rhzf_hadoop.sh  rhzf_hosts_scp.sh  rhzf_jdk.sh  rhzf_scala.sh  rhzf_spark.sh  rhzf_ssh.sh
[root@master rhzf_setup_scripts]# chmod u+x rhzf_spark.sh
[root@master rhzf_setup_scripts]# ./rhzf_spark.sh
[root@master rhzf_setup_scripts]#

6，启动spark集群

[root@master bin]# pwd
/usr/local/spark-2.1.0-bin-hadoop2.6/bin
[root@master bin]# cd ..
[root@master spark-2.1.0-bin-hadoop2.6]# cd sbin
[root@master sbin]# ls
slaves.sh         start-all.sh               start-mesos-shuffle-service.sh  start-thriftserver.sh   stop-mesos-dispatcher.sh       stop-slaves.sh
spark-config.sh   start-history-server.sh    start-shuffle-service.sh        stop-all.sh             stop-mesos-shuffle-service.sh  stop-thriftserver.sh
spark-daemon.sh   start-master.sh            start-slave.sh                  stop-history-server.sh  stop-shuffle-service.sh
spark-daemons.sh  start-mesos-dispatcher.sh  start-slaves.sh                 stop-master.sh          stop-slave.sh
[root@master sbin]# start-all.sh
starting org.apache.spark.deploy.master.Master, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.master.Master-1-master.out
worker03: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-worker03.out
worker04: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-worker04.out
worker01: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-worker01.out
worker02: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.1.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-worker02.out

显示结果如下