hadoop部署高可用(hdfs ha、yarn ha)

基础部署https://blog.csdn.net/fly0512/article/details/100863889
停止集群，安装zk （略）
最好是新集群就安装，如集群带数据，格式化会清空，那就只能做数据迁移了。
总体配置namenode 2个节点，journalnode 3个节点，yarn 2个节点，如集群规模大，适当增加。
一、配置hdfs ha ：
hdfs ha

1、hadoop-env.sh
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export HDFS_ZKFC_USER=root
export HDFS_JOURNALNODE_USER=root

2、core-site.xml
   <configuration>
    <!-- master主机名 -->
      <property>
        <name>fs.defaultFS</name>
        #集群名称不能有下划线！集群名称不能有下划线！集群名称不能有下划线！
        <value>hdfs://qingfengcluster</value>    
      </property>
    <!-- Size of read/write buffer used in SequenceFiles. -->
      <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
      </property>
        <property>
            <name>hadoop.tmp.dir</name>
            <value>/home/hadoop-3.1.2/tmp/ha</value>
	<description>A base for other temporary directories.</description> 
        </property>
      <property>
   	<name>hadoop.http.staticuser.user</name>
   	<value>root</value>
   	</property>
  <property>
   <name>ha.zookeeper.quorum</name>
   <value>qingfeng08.net:2181,qingfeng10.net:2181,qingfeng11.net:2181</value>
  </property>
  <property>
  <name>ipc.client.connect.max.retries</name>
  <value>100</value>
  <description>Indicates the number of retries a client will make to establish
      a server connection.
  </description>
 </property>
 <property>
  <name>ipc.client.connect.retry.interval</name>
  <value>10000</value>
 </property> 
    <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
    </property>
    </configuration>
	
3、hdfs-site.xml
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>2</value>
	</property>
<property>
       <name>dfs.nameservices</name>    #设置名字
       <value>qingfengcluster</value>
    </property>
<property>
  <name>dfs.ha.namenodes.qingfengcluster</name>   #设置NameNode个数，这里设置的是两个，
  <value>nn1,nn2</value>
</property>
<property>
  <name>dfs.namenode.rpc-address.qingfengcluster.nn1</name>  #远程服务的调用 
  <value>qingfeng04.net:8020</value>
</property>
<property>
  <name>dfs.namenode.rpc-address.qingfengcluster.nn2</name>
  <value>qingfeng07.net:8020</value>
</property>
<property>
  <name>dfs.namenode.http-address.qingfengcluster.nn1</name>  #浏览器访问namenode配置的IP和地址 
  <value>qingfeng04.net:50070</value>
</property>
<property>
  <name>dfs.namenode.http-address.qingfengcluster.nn2</name> #两个
  <value>qingfeng07.net:50070</value>
</property>
<property>
  <name>dfs.namenode.shared.edits.dir</name>
  <value>qjournal://qingfeng08.net:8485;qingfeng10.net:8485;qingfeng11.net:8485/mycluster</value> #journalnode设置为三个节点
</property>
<property>
  <name>dfs.client.failover.proxy.provider.qingfengcluster</name>
  <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>  #hdfs客户端找到NameNode，active类名的代理类
</property>
    <property>
      <name>dfs.ha.fencing.methods</name>
      <value>sshfence</value>
    </property>
    <property>
      <name>dfs.ha.fencing.ssh.private-key-files</name>
      <value>/root/.ssh/id_rsa</value>
    </property>
<property>
  <name>dfs.journalnode.edits.dir</name>
  <value>/home/hadoop-3.1.2/tmp/ha/journalnodeDiary</value>
</property>
<property>
# 配置自动故障转移
#加上集群名称!
   <name>dfs.ha.automatic-failover.enabled.qingfengcluster</name>
   <value>true</value>
</property>
</configuration>

4、配置文件分发
5、启动journalnode
hdfs --daemon start journalnode  #在配置好的journalnode三个节点上分别跑起来
6、在nn1节点上格式化namenode
hdfs namenode -format
hdfs zkfc -formatZK #最好格式化一下，否则可能报缺少文件错误
7、在nn1节点上启动namenode
#要起zkfc，否则会出现两个standby!要起zkfc，否则会出现两个standby!要起zkfc，否则会出现两个standby!
hdfs --daemon start zkfc
hdfs --daemon start namenode
8、在nn2节点上执行同步(要起namenode)
hdfs namenode -bootstrapStandby
cat tmp/ha/dfs/name/current/VERSION
查看ha集群ID等信息
9、启动其他服务
start-dfs.sh
jps检查进程
10、2个namenode:50070查看状态

二、配置yarn ha

vi mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <!-- 指定mr框架为yarn方式 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

    <!-- 指定mapreduce jobhistory地址 -->
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>qingfeng04.net:10020</value>
    </property>

    <!-- 任务历史服务器的web地址 -->
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>qingfeng04.net:19888</value>
    </property>

    <property>
      <name>mapreduce.application.classpath</name>
      <value>
		/home/hadoop-3.1.2/etc/hadoop,
		/home/hadoop-3.1.2/share/hadoop/common/*,
		/home/hadoop-3.1.2/share/hadoop/common/lib/*,
		/home/hadoop-3.1.2/share/hadoop/hdfs/*,
		/home/hadoop-3.1.2/share/hadoop/hdfs/lib/*,
		/home/hadoop-3.1.2/share/hadoop/mapreduce/*,
		/home/hadoop-3.1.2/share/hadoop/mapreduce/lib/*,
		/home/hadoop-3.1.2/share/hadoop/yarn/*,
		/home/hadoop-3.1.2/share/hadoop/yarn/lib/*
      </value>
    </property>

</configuration>

2、vi yarn-site.xml
<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
    <!-- 开启RM高可用 -->
    <property>
        <name>yarn.resourcemanager.ha.enabled</name>
        <value>true</value>
    </property>

    <!-- 指定RM的cluster id -->
    <property>
        <name>yarn.resourcemanager.cluster-id</name>
        <value>cluster1</value>
    </property>

    <!-- 指定RM的名字 -->
    <property>
        <name>yarn.resourcemanager.ha.rm-ids</name>
        <value>rm1,rm2</value>
    </property>

    <!-- 分别指定RM的地址 -->
    <property>
        <name>yarn.resourcemanager.hostname.rm1</name>
        <value>qingfeng08.net</value>
    </property>

    <property>
        <name>yarn.resourcemanager.hostname.rm2</name>
        <value>qingfeng10.net</value>
    </property>

    <property>
        <name>yarn.resourcemanager.webapp.address.rm1</name>
        <value>qingfeng08.net:8088</value>
    </property>
    
    <property>
        <name>yarn.resourcemanager.webapp.address.rm2</name>
        <value>qingfeng10.net:8088</value>
    </property>
   
    <!-- 指定zk集群地址 -->
    <property>
        <name>yarn.resourcemanager.zk-address</name>
        <value>qingfeng08.net:2181,qingfeng10.net:2181,qingfeng11.net:2181</value>
    </property>

    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>

    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>

    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>86400</value>
    </property>

    <!-- 启用自动恢复 -->
    <property>
        <name>yarn.resourcemanager.recovery.enabled</name>
        <value>true</value>
    </property>

    <!-- 制定resourcemanager的状态信息存储在zookeeper集群上 -->
    <property>
        <name>yarn.resourcemanager.store.class</name>
        <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
    </property>
    
    <!-- Whether virtual memory limits will be enforced for containers.  -->
    <property>
		<name>yarn.nodemanager.vmem-check-enabled</name>
		<value>false</value>
    </property>
	
    <property>
		<name>yarn.nodemanager.vmem-pmem-ratio</name>
	  	<value>5</value>
    </property>

</configuration>

3、vi hadoop-env.sh
#YARN
export YARN_RESOURCEMANAGER_USER=root
export HDFS_DATANODE_SECURE_USER=yarn
export YARN_NODEMANAGER_USER=root

4、rm1启动
start-yarn.sh