hadoop高可用的安装--3台虚拟机

解压hadoop包

1.解压hadoop包到 /usr/local里面

sudo tar -zxf hadoop-2.7.7.tar.gz -C /usr/local

2.进入到 /usr/local里面，修改名字为hadoop

cd /usr/local/
sudo mv ./hadoop-2.7.7/ ./hadoop  

3.授权当前用户hadoop拥有该文件夹的所有者权限。

sudo chown -R hadoop:hadoop ./hadoop

4.查看文件安装路径

pwd

5.配置环境变量

打开 ~/.bashrc文件

sudo vim ~/.bashrc

添加以下内容

#HADOOP_HOME
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin

使配置环境生效

 source  ~/.bashrc

查看是否生效

hadoop version

修改配置文件,以下配置文件都在 hadoop-2.7.2/etc/hadoop下

6.进入hadoop的配置文件

cd /usr/local/hadoop/etc/hadoop/

7.修改配置文件

7.1配置core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
	<!-- hdfs 的数据存储位置 -->
	<property>
		<name>hadoop.tmp.dir</name>
		<value>file:/usr/local/hadoop/tmp</value>
	</property>

	<!-- hdfs 的访问地址，ns 是 NameNode 集群的名字，在 hdfs-site.xml 文件配置 -->
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://ns</value>
	</property>

	<!-- 允许 hive 中的用户操作操纵 hdfs -->
	<property>
		<name>hadoop.proxyuser.hadoop.groups</name>
		<value>*</value>
	</property>

	<property>
		<name>hadoop.proxyuser.hadoop.hosts</name>
		<value>*</value>
	</property>

	<!-- 配置 zookeeper 地址，JournalNode 通过 zookeeper 实现功能 -->
	<property>
		<name>ha.zookeeper.quorum</name>
		<value>master:2181,slave1:2181,slave2:2181</value>
	</property>

	
</configuration>

7.2配置 yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
	<property>
             <name>yarn.nodemanager.aux-services</name>
             <value>mapreduce_shuffle</value>
	</property>

</configuration>

7.3配置hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>	
	<!-- 设置 NameNode 集群的名字为 ns -->
	<property>
		<name>dfs.nameservices</name>
		<value>ns</value>
	</property>

	<!-- 为 ns 集群中的每个 NameNode 起一个名字，分别是 nn1, nn2 -->
	<property>
		<name>dfs.ha.namenodes.ns</name>
		<value>nn1,nn2</value>
	</property>

	<!-- 分别配置每个 NameNode 的请求端口和监控页面端口 -->
	<property>
		<name>dfs.namenode.rpc-address.ns.nn1</name>
		<value>master:9000</value>
	</property>
	<property>
		<name>dfs.namenode.http-address.ns.nn1</name>
		<value>master:50070</value>
	</property>

	<property>
		<name>dfs.namenode.rpc-address.ns.nn2</name>
		<value>slave1:9000</value>
	</property>
	<property>
		<name>dfs.namenode.http-address.ns.nn2</name>
		<value>slave1:50070</value>
	</property>

	<!-- 在哪些节点启动 JournalNode 进程，用于在两个 NameNode 之间同步 fsimage 和 edits，通常是单数个 -->
	<property>
		<name>dfs.namenode.shared.edits.dir</name>
		<value>qjournal://master:8485;slave1:8485;slave2:8485/ns</value>
	</property>

	<!-- JournalNode 进程数据的存储位置 -->
	<property>
		<name>dfs.journalnode.edits.dir</name>
		<value>/usr/local/hadoop/tmp/journal</value>
	</property>

	<!-- NameNode 进程数据的存储位置  -->
	<property>
		<name>dfs.namenode.name.dir</name>
		<value>file:/usr/local/hadoop/tmp/dfs/name</value>
	</property>

	<!-- DataNode 进程数据的存储位置 -->
	<property>
		<name>dfs.datanode.data.dir</name>
		<value>file:/usr/local/hadoop/tmp/dfs/data</value>
	</property>


	<!-- 启用自动切换 -->
	<property>
		<name>dfs.ha.automatic-failover.enabled</name>
		<value>true</value>
	</property>

	<!-- 配置切换 Active 和 StrandBy 状态的类 -->
	<property>
		<name>dfs.client.failover.proxy.provider.ns</name>
		<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
	</property>

	<!-- 使用 fence 软件进行切换 -->
	<property>
		<name>dfs.ha.fencing.methods</name>
		<value>sshfence</value>
	</property>

	<!-- fence 使用需要指定私钥的地址 -->
	<property>
		<name>dfs.ha.fencing.ssh.private-key-files</name>
		<value>/home/hadoop/.ssh/id_rsa</value>
	</property>

	<!-- NameNode 等待 JournalNode 启动的超时时间 -->
	<property>
		<name>dfs.qjournal.write-txns.timeout.ms</name>
		<value>60000</value>
	</property>

</configuration>

7.4配置mapred-site.xml

由于没有mapred-site.xml，我们需要把mapred-site.xml.template修改名字

cp mapred-site.xml.template mapred-site.xml
vim mapred-site.xml

  <?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<property>
             <name>mapreduce.framework.name</name>
             <value>yarn</value> 
	</property>

</configuration>

7.5配置slaves

master
slave1
slave2

7.6配置yarn-env.sh

export JAVA_HOME=/usr/local/lib/jdk1.8.0_212

注意：
以上步骤做完之后发送到各个节，每个都需要哦

sudo scp -r hadoop slave1:`pwd`
sudo scp -r hadoop slave2:`pwd`

授权当前用户hadoop拥有该文件夹的所有者权限。

sudo chown -R hadoop:hadoop ./hadoop

8.启动zookeeper

bin/zkServer.sh start

9.启动journalnode

#在master,slave1,salve2,上启动
sbin/hadoop-daemon.sh start journalnode

10.格式化namenode

#在master上格式化
bin/hdfs namenode -format

11.把刚才格式化后的元数据拷贝到另外一个namenode上

# 一定要进入到/usr/local/hadoop/tmp中
scp -r dfs/ slave1:`pwd`

12.启动namenode

sbin/hadoop-daemon.sh start namenode

13.在没有格式化的namenode上执行：

#在salve1F上执行
bin/hdfs namenode -bootstrapStandby

14.启动第二个namenode

sbin/hadoop-daemon.sh start namenode

15.在其中一个节点上初始化zkfc(一定要启动zookeeper)

bin/hdfs zkfc -formatZK

16.重新启动hdfs，yarn

#关闭hdfs，yarn
sbin/stop-dfs.sh
sbin/stop-yarn.sh

#启动hdfs，yarn
sbin/start-dfs.sh
sbin/start-yarn.sh

#启动所有
sbin/start-all.sh

#关闭所有
sbin/stop-all.sh

17.查看节点启动是否成功

第一种：

jps

第二种：