Hadoop全分布集群搭建

1.实现伪分布搭建完成

每一台机器通过在Windows主机访问ip:50070能够成功说明伪分布以及搭建完成

2.配置NAT时间同步

首先配置头结点的配置文件

# 切换root账户
su

# 检查ntp是否安装（没有的话需要安装）
rpm -qa | grep ntp

# 安装ntp
yum install -y ntp 

# 在头节点配置ntp时间服务
# 编辑/etc/ntp.conf
vi /etc/ntp.conf

# step1:授权 192.168.137.0 - 192.168.137.255 网段上的所有主机可以从这台主机上查询和同步时间
restrict 172.20.10.0 mask 255.255.255.240 nomodify notrap

# step2:设置集群在局域网中，不使用其它互联网上的时间
# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst

# step3:设置当该节点丢失网络连接，依然可以采用本地时间作为时间服务器为集群中的其它节点提供时间同步

# CVE-2013-5211 for more details.
# Note: Monitoring will not be disabled with the limited restriction flag.

server 127.127.1.0
fudge 127.127.1.0 stratum 10

# 编辑 /etc/sysconfig/ntpd 文件： 
# 在最后一行添加 SYNC_HWCLOCK=yes ，让硬件时间与系统时间一起同步

# 重启 ntpd 服务，并设置开机自启：
service ntpd status  #查看ntpd状态
service ntpd start   #启动ntpd
service ntpd stop    #停止ntpd

注意：以上只是头结点的配置，以下是同步与头结点主机的配置

# 在主机当中设置crontab -e 下面的时间定时同步任务
*/5 * * * * /usr/sbin/utpdate mast  #这里是跟随我的mast主机同步时间，每隔五分钟同步一次

crontab -l 查看时间定时任务

# 检查是否配置完成时间同步
#在其他主机修改时间，五分钟后看是否会和自己的头结点主机时间相同
date -s '2022-9-7 9:13:20'

3.配置头结点的配置文件

修改hadoop配置文件

• hadoop-env.sh

# The java implementation to use.
export JAVA_HOME=/export/server/jdk1.8.0_171/

# The jsvc implementation to use. Jsvc is required to run secure datanodes
# that bind to privileged ports to provide authentication of data transfer
# protocol.  Jsvc is not required if SASL is configured for authentication of
# data transfer protocol using non-privileged ports.
#export JSVC_HOME=${JSVC_HOME}

#export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
export HADOOP_CONF_DIR=/export/server/hadoop-2.7.3/etc/hadoop

• hdfs-site.xml

[root@mast hadoop]# vim  hdfs-site.xml 

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>

<property>
                <name>dfs.replication</name>
                               <value>2</value>
                               <description>datenode数，默认是3，应小于datanode机器数量</description>
</property>

<property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/export/server/hadoop-2.7.3/hdfs/name</value>
<description>namenode上存储hdfs名字空间元数据</description>
</property>
<property>
    <name>dfs.datanode.data.dir</name>
<value>file:/export/server/hadoop-2.7.3/hdfs/data</value>
<description>datanode上数据块的物理存储位置</description>
</property>

</configuration>

• core-site.xml

[root@mast hadoop]# cat core-site.xml 
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
       <name>fs.default.name</name>
       <value>hdfs://mast:9000</value>
<description>指定HDFS的默认名称</description>
    </property>
    <property>
       <name>fs.defaultFS</name>
	<value>hdfs://mast:9000</value>
	<description>HDFS的URL</description>
      
       
    </property>
<property>
                <name>hadoop.tmp.dir</name>
                                <value>/export/server/hadoop-2.7.3/tmp</value>
                                                <description>节点上本地的hadoop临时文件夹</description>
                                                         </property>
</configuration>

• mapper-site.xml

[root@mast hadoop]# cat mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
                <name>mapreduce.framework.name</name>
                                <value>yarn</value>
                                                <description>指定mapreduce使用yarn框架</description>
                                                        </property>
                                                                <!-- 历史服务器端地址  -->
        <property>
                        <name>mapreduce.jobhistory.address</name>
                                        <value>mast:10020</value>
                                                </property>
                                                        <!-- 历史服务器Web端地址  -->
        <property>
                        <name>mapreduce.jobhistory.webapp.address</name>
                                        <value>mast:19888</value>
                                                </property>
                                               
</configuration>

• slaves

[root@mast hadoop]# cat slaves 
# 我这里为除主节点以外的主机名
node1
node2
node3

文件配置完成，进入/bin目录下格式化

hdfs namenode -format

文件配置完成，进入/sbin目录下启动集群

./start-all.sh

如图全分布集群跑起来啦！更可以写几个简单的mapreduce案例测试一下！