Spark 3.1 Standalone集群搭建

最新推荐文章于 2023-05-18 16:57:43 发布

程裕强

最新推荐文章于 2023-05-18 16:57:43 发布

阅读量480

点赞数

分类专栏： Spark2.x学习笔记

本文链接：https://blog.csdn.net/chengyuqiang/article/details/119611879

版权

Spark2.x学习笔记专栏收录该内容

19 篇文章 5 订阅

订阅专栏

1、下载

http://spark.apache.org/downloads.html
在这里插入图片描述

https://www.apache.org/dyn/closer.lua/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz

2、解压配置

（1）解压

[root@node2 app]# ls
nacos  spark-3.1.2-bin-hadoop3.2.tgz
[root@node2 app]# tar -zxvf spark-3.1.2-bin-hadoop3.2.tgz

（2）重命名

[root@node2 app]# mv spark-3.1.2-bin-hadoop3.2 spark-3.1.2
[root@node2 app]# cd spark-3.1.2/
[root@node2 spark-3.1.2]# ls
bin  conf  data  examples  jars  kubernetes  LICENSE  licenses  NOTICE  python  R  README.md  RELEASE  sbin  yarn
[root@node2 spark-3.1.2]# ll

（3）配置workers（旧版本是slaves）

[root@node2 spark-3.1.2]# cd conf/
[root@node2 conf]# ls
fairscheduler.xml.template  log4j.properties.template  metrics.properties.template  spark-defaults.conf.template  spark-env.sh.template  workers.template
[root@node2 conf]# vi workers.template 
[root@node2 conf]# cp workers.template workers
[root@node2 conf]# vi workers
[root@node2 conf]# cat workers
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# A Spark Worker will be started on each of the machines listed below.
node1
node2
node3
[root@node2 conf]#

（4）配置spark-env.sh

[root@node2 conf]# cp spark-env.sh.template spark-env.sh
[root@node2 conf]# vi spark-env.sh

添加一行

export JAVA_HOME=/opt/jdk1.8.0_161

（5）分发

[root@node2 app]# scp -r spark-3.1.2  node1:/app
[root@node2 app]# scp -r spark-3.1.2  node3:/app

3、启动集群

[root@node1 app]# cd spark-3.1.2
[root@node1 spark-3.1.2]# sbin/start-master.sh 
starting org.apache.spark.deploy.master.Master, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.master.Master-1-node1.out
[root@node1 spark-3.1.2]# jps
6148 PaloFe
11689 Jps
11325 Master  
[root@node1 spark-3.1.2]# sbin/start-slaves.sh 
This script is deprecated, use start-workers.sh
node1: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node1.out
node3: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node3.out
node2: starting org.apache.spark.deploy.worker.Worker, logging to /app/spark-3.1.2/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-node2.out
[root@node1 spark-3.1.2]# jps
6148 PaloFe
14983 Jps
14204 Worker
11325 Master
[root@node1 spark-3.1.2]#

[root@node2 app]# curl node1:8080
<!DOCTYPE html><html>
      <head>
        <meta http-equiv="Content-type" content="text/html; charset=utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/static/bootstrap.min.css" type="text/css"/><link rel="stylesheet" href="/static/vis-timeline-graph2d.min.css" type="text/css"/><link rel="stylesheet" href="/static/webui.css" type="text/css"/><link rel="stylesheet" href="/static/timeline-view.css" type="text/css"/><script src="/static/sorttable.js"></script><script src="/static/jquery-3.5.1.min.js"></script><script src="/static/vis-timeline-graph2d.min.js"></script><script src="/static/bootstrap.bundle.min.js"></script><script src="/static/initialize-tooltips.js"></script><script src="/static/table.js"></script><script src="/static/timeline-view.js"></script><script src="/static/log-view.js"></script><script src="/static/webui.js"></script><script>setUIRoot('')</script>
        
        <link rel="shortcut icon" href="/static/spark-logo-77x50px-hd.png"></link>
        <title>Spark Master at spark://node1:7077</title>
      </head>
      <body>
        <div class="container-fluid">
          <div class="row">
            <div class="col-12">
              <h3 style="vertical-align: middle; display: inline-block;">
                <a style="text-decoration: none" href="/">
                  <img src="/static/spark-logo-77x50px-hd.png"/>
                  <span class="version" style="margin-right: 15px;">3.1.2</span>
                </a>
                Spark Master at spark://node1:7077
              </h3>
            </div>
          </div>
          <div class="row">
            <div class="col-12">
              <div class="row">
          <div class="col-12">
            <ul class="list-unstyled">
              <li><strong>URL:</strong> spark://node1:7077</li>
              
              <li><strong>Alive Workers:</strong> 3</li>
              <li><strong>Cores in use:</strong> 12 Total,
                0 Used</li>
              <li><strong>Memory in use:</strong>
                19.9 GiB Total,
                0.0 B Used</li>
              <li><strong>Resources in use:</strong>
                </li>
              <li><strong>Applications:</strong>
                0 <a href="#running-app">Running</a>,
                0 <a href="#completed-app">Completed</a> </li>
              <li><strong>Drivers:</strong>
                0 Running,
                0 Completed </li>
              <li><strong>Status:</strong> ALIVE</li>
            </ul>
          </div>
        </div><div class="row">
          <div class="col-12">
            <span class="collapse-aggregated-workers collapse-table" onClick="collapseTable('collapse-aggregated-workers','aggregated-workers')">
              <h4>
                <span class="collapse-table-arrow arrow-open"></span>
                <a>Workers (3)</a>
              </h4>
            </span>
            <div class="aggregated-workers collapsible-table">
              <table class="table table-bordered table-sm table-striped sortable">
      <thead><th width="" class="">Worker Id</th><th width="" class="">Address</th><th width="" class="">State</th><th width="" class="">Cores</th><th width="" class="">Memory</th><th width="" class="">Resources</th></thead>
      <tbody>
        <tr>
      <td>
        <a href="http://10.17.12.158:8081">
              worker-20210811170616-10.17.12.158-36757
            </a>
      </td>
      <td>10.17.12.158:36757</td>
      <td>ALIVE</td>
      <td>4 (0 Used)</td>
      <td sorttable_customkey="6799.0">
        6.6 GiB
        (0.0 B Used)
      </td>
      <td></td>
    </tr><tr>
      <td>
        <a href="http://10.17.12.159:8081">
              worker-20210811170617-10.17.12.159-33329
            </a>
      </td>
      <td>10.17.12.159:33329</td>
      <td>ALIVE</td>
      <td>4 (0 Used)</td>
      <td sorttable_customkey="6799.0">
        6.6 GiB
        (0.0 B Used)
      </td>
      <td></td>
    </tr><tr>
      <td>
        <a href="http://10.17.12.160:8081">
              worker-20210811170617-10.17.12.160-40490
            </a>
      </td>
      <td>10.17.12.160:40490</td>
      <td>ALIVE</td>
      <td>4 (0 Used)</td>
      <td sorttable_customkey="6799.0">
        6.6 GiB
        (0.0 B Used)
      </td>
      <td></td>
    </tr>
      </tbody>
    </table>
            </div>
          </div>
        </div><div class="row">
          <div class="col-12">
            <span id="running-app" class="collapse-aggregated-activeApps collapse-table" onClick="collapseTable('collapse-aggregated-activeApps','aggregated-activeApps')">
              <h4>
                <span class="collapse-table-arrow arrow-open"></span>
                <a>Running Applications (0)</a>
              </h4>
            </span>
            <div class="aggregated-activeApps collapsible-table">
              <table class="table table-bordered table-sm table-striped sortable">
      <thead><th width="" class="">Application ID</th><th width="" class="">Name</th><th width="" class="">Cores</th><th width="" class="">Memory per Executor</th><th width="" class="">Resources Per Executor</th><th width="" class="">Submitted Time</th><th width="" class="">User</th><th width="" class="">State</th><th width="" class="">Duration</th></thead>
      <tbody>
        
      </tbody>
    </table>
            </div>
          </div>
        </div><div>
          
        </div><div class="row">
          <div class="col-12">
            <span id="completed-app" class="collapse-aggregated-completedApps collapse-table" onClick="collapseTable('collapse-aggregated-completedApps',
                'aggregated-completedApps')">
              <h4>
                <span class="collapse-table-arrow arrow-open"></span>
                <a>Completed Applications (0)</a>
              </h4>
            </span>
            <div class="aggregated-completedApps collapsible-table">
              <table class="table table-bordered table-sm table-striped sortable">
      <thead><th width="" class="">Application ID</th><th width="" class="">Name</th><th width="" class="">Cores</th><th width="" class="">Memory per Executor</th><th width="" class="">Resources Per Executor</th><th width="" class="">Submitted Time</th><th width="" class="">User</th><th width="" class="">State</th><th width="" class="">Duration</th></thead>
      <tbody>
        
      </tbody>
    </table>
            </div>
          </div>
        </div><div>
          
        </div>
            </div>
          </div>
        </div>
      </body>
    </html>[root@node2 app]#