Hadoop分布式集群搭建

---

1 资源

hadoop-2.7.7.tar.gz

提取码：swfy

2 环境

2.1 规划

IP	HOST	Zookeeper	NameNode	ZKFC	JournalNode	DataNode	ResourceManager	NodeManager	JobHistory
104.21.51.1	zk01	√	√	√	√	√		√	√
104.21.51.2	zk02	√	√	√	√	√	√	√
104.21.51.3	zk03	√			√	√	√	√

2.2 组件简介

• Zookeeper：分布式应用程序协调服务，以Fast Paxos算法为基础，实现同步服务，配置维护和命名服务等分布式应用
• NameNode：管理HDFS，监控DataNode
• ZKFC：监控NameNode的状态，并及时把状态信息写入Zookeeper。当Active的NameNode发生故障时，负责故障切换
• JournalNode：存放NameNode的editlog文件（元数据）
• DataNode：存储节点，多副本
• ResourceManager：负责各个NodeManager的资源调度
• NodeManager：管理DataNode的资源
• JobHistory：记录已经finished的mapreduce运行日志

3 安装zookeeper集群

Zookeeper分布式集群安装

4 部署hadoop集群

4.1 解压安装包

mkdir -p /hadoop
tar -zxvf hadoop-2.7.7.tar.gz -C /hadoop

4.2 设置环境变量并使之生效

vi /etc/bashrc

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.161-2.b14.el7.x86_64/jre
export HADOOP_HOME=/hadoop/hadoop-2.7.7
export PATH=$HADOOP_HOME:$PATH

source /etc/bashrc

4.3 创建数据目录

mkdir -p $HADOOP_HOME/data/ha/jn
mkdir -p $HADOOP_HOME/data/ha/tmp

4.4 配置集群

cd $HADOOP_HOME/etc/hadoop

4.4.1 core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
    <!-- 把两个NameNode的地址组装成一个集群mycluster -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://mycluster</value>
    </property>
    <!-- 指定hadoop运行时产生文件的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/hadoop/hadoop-2.7.7/data/ha/tmp</value>
    </property>
    <!-- 指定ZKFC故障自动切换转移 -->
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>zk01:2181,zk02:2181,zk03:2181</value>
    </property>
    <!-- 开启httpfs -->
    <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
    </property>
    <!-- 开启httpfs -->
    <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
    </property>
</configuration>

4.4.2 hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
    <!-- 设置dfs副本数，默认3个 -->
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
    <!-- 完全分布式集群名称 -->
    <property>
        <name>dfs.nameservices</name>