从0开始搭建hadoop伪分布式集群（四：hadoop HA）

进入HA3VM01
cd /home/hadoop/app/hadoop-2.4.1/etc/hadoop
vi hadoop-env.sh
（这里若是在之前已经配置过JAVA_HOME，则不用再进行更改）
添加JAVA_HOME
vi core-site.xml

<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://ns1/</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/home/hadoop/app/hadoop-2.4.1/tmp</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>HA3VM01:2181,HA3VM02:2181,HA3VM03:2181</value>
</property>
</configuration>

vi hdfs-site.xml

<configuration>
<property>
<name>dfs.nameservices</name>
<value>ns1</value>
</property>
<property>
<name>dfs.ha.namenodes.ns1</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns1.nn1</name>
<value>HA3VM01:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1.nn1</name>
<value>HA3VM01:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns1.nn2</name>
<value>HA3VM02:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1.nn2</name>
<value>HA3VM02:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://HA3VM01:8485;HA3VM02:8485;HA3VM03:8485/ns1</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/hadoop/app/hadoop-2.4.1/journaldata</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.ns1</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/hadoop/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>
</configuration>

vi mapred-site.xml

<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>

vi yarn-site.xml

<configuration>
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>yrc</value>
</property>
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>HA3VM01</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>HA3VM02</value>
</property>
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>HA3VM01:2181,HA3VM02:2181,HA3VM03:2181</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
</configuration>

vi slaves

HA3VM01
HA3VM02
HA3VM03

无密登录
所有虚拟机上执行
cd
rm -rf .ssh

然后进入HA3VM01的虚拟机界面
执行
ssh-keygen -t rsa
所有选择直接回车
然后执行
ssh-copy-id HA3VM01
输入yes

再执行
ssh HA3VM01
exit
发现不需要输入密码，测试成功
ssh HA3VM02
exit
ssh HA3VM03
exit

cd app
scp -r hadoop-2.4.1/ HA3VM02:/home/hadoop/app/
scp -r hadoop-2.4.1/ HA3VM03:/home/hadoop/app/

在三台机器上启动journalnode
进入sbin
./hadoop-daemon.sh start journalnode

在01上执行
bin
./hadoop namenode -format

cd /home/hadoop/app/hadoop-2.4.1
scp -r tmp/ HA3VM02:/home/hadoop/app/hadoop-2.4.1/

格式化zkfc
hdfs zkfc -formatZK

执行
cd /home/hadoop/app/zookeeper-3.4.5/bin
./zkCli.sh
ls /
有

在HA3VM02上执行
cd /home/hadoop/app/hadoop-2.4.1/sbin
yarn-daemon.sh start resourcemanager

在windows的C:\Windows\System32\drivers\etc目录下的hosts中加入

192.168.2.156 HA3VM01
192.168.2.157 HA3VM02
192.168.2.158 HA3VM03

在windows上访问namenode的界面
HA3VM01:50070
HA3VM02:50070
HA3VM03:50070

在windows上访问yarn的界面
http://ha3vm01:8088

启动集群
在三台机器上启动zookeeper
cd /home/hadoop/app/zookeeper-3.4.5/bin
./zkServer.sh start

在01执行start-dfs.sh
在01执行start-yarn.sh
在02执行
cd /home/hadoop/app/hadoop-2.4.1/sbin
yarn-daemon.sh start resourcemanager

关闭集群
在02执行
cd /home/hadoop/app/hadoop-2.4.1/sbin
yarn-daemon.sh stop resourcemanager

在01执行stop-yarn.sh
在01执行stop-dfs.sh

在三台机器上关闭zookeeper
cd /home/hadoop/app/zookeeper-3.4.5/bin
./zkServer.sh stop