Hadoop集群搭建

安装版本

04 August, 2017: Release 2.7.4

安装参考文档

配置参考文档

注意事项

  1. hostname 不要加特殊字符串,只用字母和数字(在停止的时候一直读取到的用户名是小写的,具体原因不清楚)
  2. 本次安装中 master 是主机 11.11.11.4,slaves 是主机 11.11.11.3,11.11.11.9
  3. 启动之后进程

    • slaves

      [root@host3 hadoop-2.7.4]# jps -l
      1225 org.apache.hadoop.hdfs.server.datanode.DataNode
      1343 org.apache.hadoop.yarn.server.nodemanager.NodeManager
      
      [root@host9 hadoop-2.7.4]# jps -l
      1237 org.apache.hadoop.hdfs.server.datanode.DataNode
      1355 org.apache.hadoop.yarn.server.nodemanager.NodeManager
    • master

      [root@host4 hadoop-2.7.4]# jps -l
      1410 org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
      1256 org.apache.hadoop.hdfs.server.namenode.NameNode
  4. web管理URL http://11.11.11.4:50070
  5. 启动和停止脚本的运行都是在master主机上运行,不在slaves所在主机做任何命令
  6. stop-all.sh
[root@host4 hadoop-2.7.4]# ./sbin/stop-all.sh
This script is Deprecated. Instead use stop-dfs.sh and stop-yarn.sh
Stopping namenodes on [host4]
host4: stopping namenode
11.11.11.3: stopping datanode
11.11.11.9: stopping datanode
Stopping secondary namenodes [host4]
host4: stopping secondarynamenode
stopping yarn daemons
stopping resourcemanager
11.11.11.9: stopping nodemanager
11.11.11.3: stopping nodemanager
no proxyserver to stop
  1. start-all.sh
[root@host4 hadoop-2.7.4]# ./sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [host4]
host4: starting namenode, logging to /usr/local/hadoop-2.7.4/logs/hadoop-root-namenode-host4.out
11.11.11.9: starting datanode, logging to /usr/local/hadoop-2.7.4/logs/hadoop-root-datanode-host9.out
11.11.11.3: starting datanode, logging to /usr/local/hadoop-2.7.4/logs/hadoop-root-datanode-host3.out
Starting secondary namenodes [host4]
host4: starting secondarynamenode, logging to /usr/local/hadoop-2.7.4/logs/hadoop-root-secondarynamenode-host4.out
starting yarn daemons
starting resourcemanager, logging to /usr/local/hadoop-2.7.4/logs/yarn-root-resourcemanager-host4.out
11.11.11.3: starting nodemanager, logging to /usr/local/hadoop-2.7.4/logs/yarn-root-nodemanager-host3.out
11.11.11.9: starting nodemanager, logging to /usr/local/hadoop-2.7.4/logs/yarn-root-nodemanager-host9.out
  1. jps
#master
[root@host4 hadoop-2.7.4]# jps -l
4706 sun.tools.jps.Jps
4083 org.apache.hadoop.hdfs.server.namenode.NameNode
4275 org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode
4440 org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
1710 org.apache.zookeeper.server.quorum.QuorumPeerMain

#slave
[root@host9 hadoop-2.7.4]# jps -l
2291 org.apache.hadoop.yarn.server.nodemanager.NodeManager
2438 sun.tools.jps.Jps
2185 org.apache.hadoop.hdfs.server.datanode.DataNode
1518 org.apache.zookeeper.server.quorum.QuorumPeerMain

#slave
[root@host3 hadoop-2.7.4]# jps -l
2161 org.apache.hadoop.hdfs.server.datanode.DataNode
1493 org.apache.zookeeper.server.quorum.QuorumPeerMain
2267 org.apache.hadoop.yarn.server.nodemanager.NodeManager
2399 sun.tools.jps.Jps
  1. netstat
#slave
[root@host3 hadoop-2.7.4]# netstat -anto
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address               Foreign Address             State       Timer
tcp        0      0 0.0.0.0:22                  0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 127.0.0.1:25                0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 0.0.0.0:50010               0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 0.0.0.0:50075               0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 127.0.0.1:37985             0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 0.0.0.0:50020               0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 11.11.11.3:22               11.11.11.4:43435            ESTABLISHED keepalive (5907.23/0/0)
tcp        0      0 11.11.11.3:59127            11.11.11.4:9000             ESTABLISHED keepalive (6994.07/0/0)
tcp        0      0 :::36914                    :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::22                       :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::1:25                      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::13562                    :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::60514                    :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.3:2181      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::8040                     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::8042                     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.3:3888      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.3:3888      ::ffff:11.11.11.9:50710     ESTABLISHED off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.3:3888      ::ffff:11.11.11.4:41540     ESTABLISHED off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.3:51059     ::ffff:11.11.11.4:2888      ESTABLISHED off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.3:50958     ::ffff:11.11.11.4:18025     ESTABLISHED keepalive (7007.43/0/0)

#slave
[root@host9 hadoop-2.7.4]# netstat -anto
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address               Foreign Address             State       Timer
tcp        0      0 127.0.0.1:53489             0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 0.0.0.0:22                  0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 127.0.0.1:25                0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 0.0.0.0:50010               0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 0.0.0.0:50075               0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 0.0.0.0:50020               0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 11.11.11.9:22               11.11.11.4:33271            ESTABLISHED keepalive (5833.68/0/0)
tcp        0      0 11.11.11.9:45797            11.11.11.4:9000             ESTABLISHED keepalive (6929.57/0/0)
tcp        0      0 :::22                       :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::36729                    :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::1:25                      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::13562                    :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.9:2181      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::8040                     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::8042                     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::42572                    :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.9:3888      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.9:35379     ::ffff:11.11.11.4:2888      ESTABLISHED off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.9:57459     ::ffff:11.11.11.4:18025     ESTABLISHED keepalive (6942.69/0/0)
tcp        0      0 ::ffff:11.11.11.9:52517     ::ffff:11.11.11.4:3888      ESTABLISHED off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.9:50710     ::ffff:11.11.11.3:3888      ESTABLISHED off (0.00/0/0)

#master
[root@host4 hadoop-2.7.4]# netstat -anto
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address               Foreign Address             State       Timer
tcp        0      0 0.0.0.0:50070               0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 0.0.0.0:22                  0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 127.0.0.1:25                0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 11.11.11.4:9000             0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 11.11.11.4:50090            0.0.0.0:*                   LISTEN      off (0.00/0/0)
tcp        0      0 11.11.11.4:22               192.168.0.99:54470          ESTABLISHED keepalive (5769.92/0/0)
tcp        0      0 11.11.11.4:9000             11.11.11.9:45797            ESTABLISHED keepalive (6909.35/0/0)
tcp        0      0 11.11.11.4:22               192.168.0.99:54809          ESTABLISHED keepalive (6717.90/0/0)
tcp        0      0 11.11.11.4:9000             11.11.11.3:59127            ESTABLISHED keepalive (6909.26/0/0)
tcp        0      0 11.11.11.4:22               192.168.0.99:54472          ESTABLISHED keepalive (5772.03/0/0)
tcp      160      0 11.11.11.4:22               192.168.0.99:54469          ESTABLISHED keepalive (5767.57/0/0)
tcp        0      0 11.11.11.4:33271            11.11.11.9:22               ESTABLISHED keepalive (5813.36/0/0)
tcp        0      0 11.11.11.4:33944            11.11.11.4:9000             TIME_WAIT   timewait (25.67/0/0)
tcp        0      0 11.11.11.4:43435            11.11.11.3:22               ESTABLISHED keepalive (5822.36/0/0)
tcp        0      0 :::22                       :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:18040     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::1:25                      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 :::47674                    :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:18141     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:2181      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:18088     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:2888      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:18025     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:18030     :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:3888      :::*                        LISTEN      off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:3888      ::ffff:11.11.11.9:52517     ESTABLISHED off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:2888      ::ffff:11.11.11.3:51059     ESTABLISHED off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:18025     ::ffff:11.11.11.9:57459     ESTABLISHED keepalive (6922.48/0/0)
tcp        0      0 ::ffff:11.11.11.4:2888      ::ffff:11.11.11.9:35379     ESTABLISHED off (0.00/0/0)
tcp        0      0 ::ffff:11.11.11.4:18025     ::ffff:11.11.11.3:50958     ESTABLISHED keepalive (6922.61/0/0)
tcp        0      0 ::ffff:11.11.11.4:41540     ::ffff:11.11.11.3:3888      ESTABLISHED off (0.00/0/0)

安装步骤

  1. 安装JDK
  2. 配置免密登录
    • 免密登录即将本机通过ssh-keygen -t rsa 生成的 ~/.ssh/id_rsa.pub 文件追加到远程要登录的 ~/.ssh/authorized_keys文件尾部即可免密登录
  3. 安装hadoop
    • 下载解压
    • 配置HADOOP_HOME(/etc/profile,完成之后用source /etc/profile使其生效)
    • 配置相关信息
    • 启动服务

启动方式

  1. 格式化namenode(仅在master所在机子)
cd $HADOOP_HOME
./bin/hdfs namenode -format <cluster_name>
  1. 启动hdfs
cd $HADOOP_HOME
./sbin/hadoop-daemon.sh start namenode
./sbin/hadoop-daemons.sh start datanode
上面两条命令可用用下面一条代替
cd $HADOOP_HOME
./sbin/start-dfs.sh
  1. 启动yarn
cd $HADOOP_HOME
./sbin/yarn-daemon.sh start resourcemanager
./sbin/yarn-daemons.sh start nodemanager
上面两条命令可用用下面一条代替
cd $HADOOP_HOME
./sbin/start-yarn.sh

停止服务

将启动中的start改成stop即可

配置文件

所有配置文件所在目录 $HADOOP_HOME/etc/hadoop

指定JDK的位置

修改hadoop-env.sh和yarn-env.sh文件中的JAVA_HOME的值为JDK安装目录

slaves

11.11.11.3
11.11.11.4

core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://11.11.11.4:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:///data/hadoop/tmp</value>
        <description>A base for other temporary directories.</description>
    </property>
</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:///data/hadoop/hdfs/name</value>
        <final>true</final>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:///data/hadoop/hdfs/data</value>
        <final>true</final>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>11.11.11.4:50090</value>
    </property>
    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
    </property>
</configuration>

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
 <property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
 </property>
</configuration>

yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>
    <!-- Site specific YARN configuration properties -->
    <property>
        <name>yarn.resourcemanager.address</name>
        <value>11.11.11.4:18040</value>
    </property>
    <property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>11.11.11.4:18030</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>11.11.11.4:18088</value>
    </property>
    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>11.11.11.4:18025</value>
    </property>
    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>11.11.11.4:18141</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>11.11.11.4</value>
    </property>
</configuration>
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值