配置Hadoop-HA集群_hadoop ha 集群配置-CSDN博客

本文链接：https://blog.csdn.net/zmzdmx/article/details/108473706

1.环境准备

修改IP地址与hostname

2.修改配置文件

2.1修改core-site.xml

<configuration>
<!--指定hdfs的nameserve为ns-->
 <property>
  <name>fs.defaultFS</name>
  <value>hdfs://ns</value>
 </property>
<!--指定hadoop临时目录-->
 <property>
  <name>hadoop.tmp.dir</name>
  <value>/opt/hadoop/tmp</value>
 </property>
<!--指定zookeeper的地址-->
 <property>
  <name>ha.zookeeper.quorum</name>
  <value>hadoop001:2181,hadoop002:2181,hadoop003:2181</value>
 </property>
<property>
   <name>hadoop.proxyuser.root.hosts</name>
   <value>*</value>
 </property>
<property>
   <name>hadoop.proxyuser.root.groups</name>
   <value>*</value>
 </property>
<property>
    <name>hadoop.native.lib</name>
    <value>false</value>
    <description>Should native hadoop libraries, if present, be used.
    </description>
</property>
</configuration>

2.2修改hdfs-site.xml

<configuration>
<!--节点数,ha中可不配置-->
  <property>
    <name>dfs.replication</name>
    <value>3</value>
  </property>
<!--指定名称服务,与core-site.xml中的一致-->
  <property>
    <name>dfs.nameservices</name>
    <value>ns</value>
  </property>
<!--指定名称服务下的namenode节点-->
  <property>
    <name>dfs.ha.namenodes.ns</name>
    <value>hadoop001,hadoop002</value>
  </property>
<!--rpc地址,9000端口-->
  <property>
    <name>dfs.namenode.rpc-address.ns.hadoop001</name>
    <value>hadoop001:9000</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-address.ns.hadoop002</name>
    <value>hadoop002:9000</value>
  </property>
<!--http地址,50070端口,页面地址-->
  <property>
    <name>dfs.namenode.http-address.ns.hadoop001</name>
    <value>hadoop001:50070</value>
  </property>
  <property>
    <name>dfs.namenode.http-address.ns.hadoop002</name>
    <value>hadoop002:50070</value>
  </property>
<!--namenode在jornalnode(datanode)上存放数据的端口8485-->
  <property>
    <name>dfs.namenode.shared.edits.dir</name>
    <value>qjournal://hadoop001:8485;hadoop002:8485;hadoop003:8485/ns</value>
  </property>
<!--开启namenode失败自动切换-->
  <property>
     <name>dfs.ha.automatic-failover.enabled</name>
     <value>true</value>
  </property>
<!--设置namenode切换方式,ns为core中的命名空间-->
  <property> 
    <name>dfs.client.failover.proxy.provider.ns</name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
  </property>
<!-- 配置隔离机制方法，多个机制用换行分割，即每个机制占用一行-->
  <property>
    <name>dfs.ha.fencing.methods</name>
    <value>
           sshfence
           shell(/bin/true)
    </value>
  </property>
 <!-- 配置sshfence隔离机制超时时间 -->
  <property>
    <name>dfs.ha.fencing.ssh.connect-timeout</name>
    <value>30000</value>
  </property>
  <property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
  </property>
<!--当使用高可用HA配置集群,则不需要secondarynamenode
  <property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>192.168.184.35:50090</value>
  </property>
-->
</configuration>

2.3修改 yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
<!--<property>
  <name>yarn.resourcemanager.hostname</name>
  <value>hadoop002</value>
 </property>-->
<!--开启rm高可用-->
<property>
  <name>yarn.resourcemanager.ha.enabled</name>
  <value>true</value>
 </property>

<!--指定rm的集群id-->
<property>
  <name>yarn.resourcemanager.cluster-id</name>
  <value>cluster-id</value>
 </property>

<!--指定rm的名字-->
<property>
  <name>yarn.resourcemanager.ha.rm-ids</name>
  <value>rm1,rm2</value>
 </property>

<!--分别指定rm的地址-->
<property>
 <name>yarn.resourcemanager.hostname.rm1</name>
 <value>hadoop001</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>hadoop002</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
 <value>hadoop001:8088</value>
</property>
 <property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>hadoop002:8088</value>
</property>

 <!-- 指定zk集群地址 -->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>hadoop001:2181,hadoop002:2181,hadoop003:2181</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

<property>
  <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
  <value>org.apache.hadoop.mapred.ShuffleHandler</value>
 </property>
<!-- 日志聚集功能使用 -->
<property>
  <name>yarn.log-aggregation-enable</name>
  <value>true</value>
</property>
<!-- 日志保留时间设置7天 -->
<property>
  <name>yarn.log-aggregation.retain-seconds</name>
  <value>604800</value>
</property>

</configuration>

2.4分发配置文件

scp -r hadoop hadoop002:$PWD

scp -r hadoop hadoop003:$PWD

3.启动集群

删除logs/tmp 并停止hadoop集群

rm -rf logs/
rm -rf tmp/
stop-all.sh

启动 JournalNode

hadoop-daemons.sh start journalnode

格式化 Hadoop 集群

hadoop namenode -format

复制 hadoop001临时目录tmp到hadoop002上

scp tmp/ hadoop002:$PWD

启动zookeeper

bin/zkServer.sh start

启动集群

start-all.sh

在备用节点上启动备用resourcemanager

yarn-daemon.sh start resourcemanager

格式化zkfc（三台都需要）

 hdfs zkfc -formatZK

启动zkfc

hadoop-daemon.sh start zkfc

jobhistory的启动与停止（一台不行就换一台）

1. 启动： 在hadoop/sbin/目录下执行
   mr-jobhistory-daemon.sh start historyserver
2. 停止：在hadoop/sbin/目录下执行
   mr-jobhistory-daemon.sh stop historyserver

4.测试集群是否部署成功

如果出现以下进程代表完成

在这里插入图片描述