Hadoop 高可用 全分布式 hdfs 集群，4个重要文件配置

core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<!--hdfs的默认url路径-->
<property>
	<name>fs.defaultFS</name>
        <value>hdfs://xuan/</value>
</property>
<!--hdfs的临时文件位置-->
<property>
	<name>hadoop.tmp.dir</name>
	<value>/opt/app/hadoop-2.7.7/hadoopdata/tmp</value>
</property>
<!--数据传输的缓存大小-->
<property>
        <name>io.file.buffer.size</name>
        <value>4096</value>
</property>
<!-- 指定zookeeper地址 -->
        <property>
                <name>ha.zookeeper.quorum</name>
                <value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
        </property>

</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<!--配置文件的副本数-->
<property>
        <name>dfs.replication</name>
        <value>3</value>
</property>
<!--配置块的大小-->
<property>
	<name>dfs.blocksize</name>
	<value>134217728</value>
</property>
<!--指定namenode的web ui通信地址-->  
  <property>
    <name>dfs.namenode.http-address.xuan.nn1</name>
    <value>hadoop01:50070</value>
  </property>
  <property>
    <name>dfs.namenode.http-address.xuan.nn2</name>
    <value>hadoop02:50070</value>
  </property>
<!--配置namenode的数据存放路径-->
<property>
        <name>dfs.namenode.name.dir</name>
        <value>/opt/app/hadoop-2.7.7/hadoopdata/dfs/name</value>
</property>
<!--配置datanode的数据存放路径-->
<property>
        <name>dfs.datanode.data.dir</name>
        <value>/opt/app/hadoop-2.7.7/hadoopdata/dfs/data</value>
</property>
<!--文件的检测目录-->
<property>
        <name>fs.checkpoint.dir</name>
        <value>/opt/app/hadoop-2.7.7/hadoopdata/dfs/cname</value>
</property>
<!--日志的检测目录-->
<property>
        <name>fs.checkpoint.edits.dir</name>
        <value>/opt/app/hadoop-2.7.7/hadoopdata/dfs/cname</value>
</property>
<!--是否开启hdfs的文件系统权限-->
<property>
        <name>dfs.permissions</name>
        <value>false</value>
</property>
<!--是否开启webhdfs的权限-->
<property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
</property>

<!--指定hdfs的虚拟服务名-->
  <property>
    <name>dfs.nameservices</name>
    <value>xuan</value>
  </property>
<!--指定hdfs的虚拟服务名下的namenode-->
  <property>
    <name>dfs.ha.namenodes.xuan</name>
    <value>nn1,nn2</value>
  </property>  
  <!--指定namenode的内部通信地址-->
  <property>
    <name>dfs.namenode.rpc-address.xuan.nn1</name>
    <value>hadoop01:9000</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-address.xuan.nn2</name>
    <value>hadoop02:9000</value>
  </property>
 <!--指定journalnode数据共享目录-->
  <property>
    <name>dfs.namenode.shared.edits.dir</name>
    <value>qjournal://hadoop01:8485;hadoop02:8485;hadoop03:8485/xuan</value>
  </property>
 <!--存放journalnode本地存放目录-->
   <property>
    <name>dfs.journalnode.edits.dir</name>
    <value>/opt/app/hadoop-2.7.7/hadoopdata/journaldata</value>
  </property>
  <!--开启namenode失败是否自动切换-->
   <property>
    <name>dfs.ha.automatic-failover.enabled</name>
    <value>true</value>
  </property>
 <!--指定namanode失败进行自动切换的主类-->
   <property>
     <name>dfs.client.failover.proxy.provider.xuan</name>
     <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
   </property>
   <!--指定脑裂时，采用某种方式杀死期中一个-->
  <!--防止多个namenode,同active(脑裂)-->
   <property>
    <name>dfs.ha.fencing.methods</name>
    <value>sshfence</value>
  </property>
<property>
    <name>dfs.ha.fencing.ssh.private-key-files</name>
    <value>/root/.ssh/id_rsa</value>
  </property>  
   <property>
    <name>dfs.ha.fencing.ssh.connect-timeout</name>
    <value>30000</value>
  </property>






</configuration>

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<!--配置mapreduce程序的执行模式-->
<property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
	<final>true</final>
</property>
<!--历史服务的通信地址-->
<property>
        <name>mapreduce.jobhistory.address</name>
        <value>hadoop01:10020</value>
</property>
<!--历史服务的web ui地址-->
<property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>hadoop01:19888</value>
</property>

</configuration>

yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
<!--配置mapreduce的shuffle过程-->
<!-- 开启RM高可用 -->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<!-- 指定RM的cluster id -->
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>yrc</value>
</property>
<!-- 指定RM的名字 -->
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<!-- 分别指定RM的地址 -->
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>hadoop01</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>hadoop02</value>
</property>
<!-- 指定zk集群地址 -->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>hadoop01:8030</value>
    </property>
    <property>
        <name>yarn.resourcemanager.address</name>
        <value>hadoop01:8032</value>
    </property>
    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>hadoop01:8033</value>
    </property>
    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>hadoop01:8031</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>hadoop01:8088</value>
    </property>

</configuration>