ubuntu 16.04 搭建hadoop HA完全分布式系统

根据参考文件，本文的布置环境如下：
VMWARE 14
双核单CPU
4G内存（建议一定要4G）
三个虚拟机

hadoop vesion 2.7.7

布置方案：
先设置zookeeper，然后对两个namenode进行format
然后zookeeperformat
然后run 起来

配置文件

core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
        <property>
             <name>hadoop.tmp.dir</name>
             <value>/usr/local/hadoop/tmp</value>
             <description>Abase for other temporary directories.</description>
        </property>
        <property>
             <name>fs.defaultFS</name>
             <value>hdfs://ns1</value>
        </property>
	<property>
	     <name>ha.zookeeper.quorum</name>
	     <value>hadoop11:2181,hadoop12:2181,hadoop13:2181</value>
	</property>
</configuration>

yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Site specific YARN configuration properties -->

<configuration> 
<!-- Site specific YARN configuration properties -->
    <property> 
        <name>yarn.resourcemanager.ha.enabled</name> 
        <value>true</value> 
    </property>
<!--hehehe -->
    <property> 
        <name>yarn.resourcemanager.cluster-id</name> 
        <value>yrc</value> 
    </property>
    <property> 
        <name>yarn.resourcemanager.ha.rm-ids</name> 
        <value>rm1,rm2</value> 
    </property>
    <property> 
        <name>yarn.resourcemanager.hostname.rm1</name> 
        <value>hadoop11</value> 
    </property> 
    <property> 
        <name>yarn.resourcemanager.hostname.rm2</name> 
        <value>hadoop12</value> 
    </property> 
    <property> 
        <name>yarn.resourcemanager.zk-address</name> 
        <value>hadoop11:2181,hadoop12:2181,hadoop13:2181</value> 
    </property> 
    <property> 
        <name>yarn.resourcemanager.aux-services</name> 
        <value>mapreduce_shuffle</value> 
    </property> 
</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
        <property>
             <name>dfs.replication</name>
             <value>3</value>
        </property>
        <property>
             <name>dfs.nameservices</name>
             <value>ns1</value>
        </property>
        <property>
             <name>dfs.ha.namenodes.ns1</name>
             <value>nn1,nn2</value>
        </property>
	<property>
	     <name>dfs.namenode.rpc-address.ns1.nn1</name>
	     <value>hadoop11:9000</value>
	</property>
	<property>
	     <name>dfs.namenode.http-address.ns1.nn1</name>
	     <value>hadoop11:50070</value>
	</property>
	<property>
	     <name>dfs.namenode.rpc-address.ns1.nn2</name>
	     <value>hadoop12:9000</value>
	</property>
	<property>
	     <name>dfs.namenode.http-address.ns1.nn2</name>
	     <value>hadoop12:50070</value>
	</property>
	<property>
	     <name>dfs.namenode.shared.edits.dir</name>
	     <value>qjournal://hadoop11:8485;hadoop12:8485;hadoop13:8485/ns1</value>
	</property>
	<property>
	     <name>dfs.journalnode.edits.dir</name>
	     <value>/usr/local/hadoop/journaldata</value>
	</property>
	<property>
	     <name>dfs.ha.automatic-failover.enabled</name>
	     <value>true</value>
	</property>
	<property>
		<name>dfs.client.failover.proxy.provider.ns1</name>
		<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
	</property>
	<property>
	     <name>dfs.ha.fencing.methods</name>
	     <value>sshfence</value>
	</property>
	<property>
	     <name>dfs.ha.fencing.ssh.private-key-files</name>
	     <value>/root/.ssh/id_rsa</value>
	</property>
	<property>
	     <name>dfs.ha.fencing.ssh.connect-timeout</name>
	     <value>30000</value>
	</property>
</configuration>

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>

具体步骤

打开zookeeper
主node做format，第二个namenode把tmp文件夹拷贝过去
格式化zookeeper
start-dfs.sh
start-yarn.sh

访问一下50070
8088节点看下，两个都能看到3个node，那就对了

然后试一下传文件，如果传文件成功了才算ok
hadoop fs -put /etc/profile /
hadoop fs -ls /

命令后面有空再来写一写，夜深了

参考文献：
hadoop+spark生态系统实战与指南

小技巧：
拷贝文件到对应机器上
csp -r xxx hadoop12:/usr/local/hadoop