Hadoop hdfs HA集群部署_hdfs ha部署-CSDN博客

本文链接：https://blog.csdn.net/weixin_43876317/article/details/108828365

hdfs + zookeeper HA集群部署

1.3台服务器节点免密

修改hosts文件

分别在 172.21.36.35，172.21.36.36，172.21.36.37

vi /etc/hosts
172.21.36.35 node01
172.21.36.36 node02
172.21.36.37 node03

#三台机器互相免密登录
#1、三台机器执行以下命令
[root@node1 ~]# ssh-keygen -t rsa
[root@node2 ~]# ssh-keygen -t rsa
[root@node3 ~]# ssh-keygen -t rsa
#2、进入/.ssh/目录 该目录存放密钥对
[root@node1 ~]# cd ~/.ssh/
#3、新建authorized_keys并设置权限
[root@node1 .ssh]# touch authorized_keys
[root@node1 .ssh]# chmod 600 authorized_keys 
#4、公钥追加 authorized_keys 
[root@node1 .ssh]# cat id_rsa.pub >> authorized_keys
#5、另外两台机器也执行如下操作
#node2
[root@node2 ~]# cd ~/.ssh/
[root@node2 .ssh]# touch authorized_keys
[root@node2 .ssh]# chmod 600 authorized_keys 
[root@node2 .ssh]# cat id_rsa.pub >> authorized_keys
#node3
[root@node3 ~]# cd ~/.ssh/
[root@node3 .ssh]# touch authorized_keys
[root@node3 .ssh]# chmod 600 authorized_keys 
[root@node3 .ssh]# cat id_rsa.pub >> authorized_keys
#6、把node2的公钥追加到node1的authorized_keys
[root@node2 .ssh]# scp id_rsa.pub node1:/home/
[root@node1 .ssh]# cat ../../home/id_rsa.pub >> ./authorized_keys
#7、把node1的公钥追加到node2的authorized_keys
[root@node1 .ssh]# scp id_rsa.pub node2:/home/
[root@node2 .ssh]# cat ../../home/id_rsa.pub >> ./authorized_keys
#8、把node1的公钥追加到node3的authorized_keys
[root@node1 .ssh]# scp id_rsa.pub node3:/home/
[root@node3 .ssh]# cat ../../home/id_rsa.pub >> ./authorized_keys
#9、把node2的公钥追加到node3的authorized_keys
[root@node2 .ssh]# scp id_rsa.pub node3:/home/
[root@node3 .ssh]# cat ../../home/id_rsa.pub >> ./authorized_keys
#10、把node3的公钥追加到node1的authorized_keys
[root@node3 .ssh]# scp id_rsa.pub node1:/home/
[root@node1 .ssh]# cat ../../home/id_rsa.pub >> ./authorized_keys
#10、把node3的公钥追加到node2的authorized_keys
[root@node3 .ssh]# scp id_rsa.pub node1:/home/
[root@node2 .ssh]# cat ../../home/id_rsa.pub >> ./authorized_keys

2.部署jdk

[root@node01 ~] tar -zxvf /opt/tools/jdk-8u221-linux-x64.tar.gz -C /usr/local/
[root@node01 ~] mv /usr/local/jdk-8u221   /usr/local/jdk 
#设置环境变量
[root@node01 ~] vi /etc/profile
#Set Java Enviornment
export JAVA_HOME=/usr/local/jdk1.8.0_221
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
#分发到node02，node03
[root@node01 ~] scp -r /usr/local/jdk  node02:/usr/local
[root@node01 ~] scp -r /usr/local/jdk  node03:/usr/local
#node01，node02，node03节点加载生效
[root@node01 ~] source /etc/profile
[root@node02 ~] source /etc/profile
[root@node03 ~] source /etc/profile

3.部署zookeeper集群

[root@node01 ~]# tar xf /opt/tools/zookeeper-3.4.14.tar.gz -C /opt/
[root@node01 ~]# mv /opt/zookeeper-3.4.14  /opt/zookeeper

修改配置文件

[root@node01 ~]# cp /opt/zookeeper/conf/zoo_sample.cfg   /opt/zookeeper/conf/zoo.cfg
[root@node01 ~]# egrep -v "^#|^$" /opt/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/zookeeper/data
dataLogDir=/opt/zookeeper/log
clientPort=2181
server.1=node01:2888:3888
server.2=node02:2888:3888
server.3=node03:2888:3888

#创建zookeeper数据目录，日志目录
[root@node01 ~]# mkdir /opt/zookeeper/data -p
[root@node01 ~]# mkdir /opt/zookeeper/log -p

#分发到node02，node03
[root@node01 ~]# scp -r /opt/zookeeper node02:/opt/
[root@node01 ~]# scp -r /opt/zookeeper node03:/opt

修改zookeeper节点id(3台节点)

[root@node01 ~]# echo 1 >/opt/zookeeper/data/myid
[root@node01 ~]# echo 2 >/opt/zookeeper/data/myid
[root@node01 ~]# echo 3 >/opt/zookeeper/data/myid

启动3台zookeeper节点

[root@node01 ~]# /opt/zookeeper/bin/zkServer.sh start
[root@node02 ~]# /opt/zookeeper/bin/zkServer.sh start
[root@node03 ~]# /opt/zookeeper/bin/zkServer.sh start

查看zookeeper节点状态

[root@node01 ~]# /opt/zookeeper/bin/zkServer.sh status 
[root@node02 ~]# /opt/zookeeper/bin/zkServer.sh status 
[root@node03 ~]# /opt/zookeeper/bin/zkServer.sh status

4.部署hadoop 集群

1.解压安装包

node01操作

[root@node01 ~]# tar xf /opt/tools/hadoop-3.2.1.tar.gz -C /opt/
[root@node01 ~]# mv /opt/hadoop-3.2.1.tar.gz  /opt/hadoop
#替换文件内容
[root@node01 ~]# vi /opt/hadoop/etc/hadoop/workers
node01
node02
node03
[root@node01 ~]# vi /opt/hadoop/etc/hadoop/core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
<name>ipc.client.connect.max.retries</name>
<value>100</value>
<description>
Indicates the number of retries a client will make to establisha server connection.
</description>
</property>

 <property>
 <!-- 指定namenode的hdfs协议文件系统的通信地址 -->
 <name>fs.defaultFS</name>
 <value>hdfs://puegg</value>
 </property>
 <property>
 <!-- 指定hadoop集群存储临时文件的目录 -->
 <name>hadoop.tmp.dir</name>
 <value>/opt/hadoop/tmp</value>
 </property>
 <property>
 <!-- 指定hadoop集群管理用户 -->
 <name>hadoop.http.staticuser.user</name>
 <value>root</value>
 </property>
 <property>
 <!-- ZooKeeper集群的地址 -->
 <name>ha.zookeeper.quorum</name>
 <value>node01:2181,node02:2181,node03:2181</value>
 </property>
 <property>
 <!-- ZKFC连接到ZooKeeper超时时长 -->
 <name>ha.zookeeper.session-timeout.ms</name>
 <value>10000</value>
 </property>
</configuration>
[root@node01 ~]# vi  /opt/hadoop/etc/hadoop/hadoop-env.sh
export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
export JAVA_HOME=/usr/local/jdk
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_ZKFC_USER=root
export HDFS_JOURNALNODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

[root@node01 ~]# vi /opt/hadoop/etc/hadoop/hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>

<property>
<!-- 集群服务的逻辑名称 -->
  <name>dfs.nameservices</name>
  <value>puegg</value>
</property>

<property>
<!-- NameNode ID列表-->
  <name>dfs.ha.namenodes.puegg</name>
  <value>nn1,nn2</value>
</property>
<property>
<!-- nn1的RPC通信地址 -->
  <name>dfs.namenode.rpc-address.puegg.nn1</name>
  <value>node01:8020</value>
</property>
<property>
<!-- nn2的RPC通信地址 -->
  <name>dfs.namenode.rpc-address.puegg.nn2</name>
  <value>node02:8020</value>
</property>
<property>
<!-- nn1的http通信地址 -->
  <name>dfs.namenode.http-address.puegg.nn1</name>
  <value>node01:9870</value>
</property>
<property>
<!-- nn2的http通信地址 -->
  <name>dfs.namenode.http-address.puegg.nn2</name>
  <value>node02:9870</value>
</property>

<property>
  <!-- NameNode元数据在JournalNode上的共享存储目录 -->
  <name>dfs.namenode.shared.edits.dir</name>
  <value>qjournal://node01:8485;node02:8485;node03:8485/puegg</value>
</property>

<property>
<!-- 访问代理类，用于确定当前处于Active状态的NameNode -->
  <name>dfs.client.failover.proxy.provider.puegg</name>
  <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>

<property>
<!-- 配置隔离机制，确保在任何给定时间只有一个NameNode处于活动状态 -->
  <name>dfs.ha.fencing.methods</name>
 <value>
  sshfence
  shell(/bin/true)
 </value>


</property>
<property>
<!-- 使用sshfence机制时需要ssh免密登录 -->
  <name>dfs.ha.fencing.ssh.private-key-files</name>
  <value>/root/.ssh/id_rsa</value>
</property>

<property>
<!-- Journal Edit Files的存储目录 -->
  <name>dfs.journalnode.edits.dir</name>
  <value>/opt/hadoop/journalnode/data</value>
</property>

<property>
<!-- 开启故障自动转移 -->
  <name>dfs.ha.automatic-failover.enabled</name>
  <value>true</value>
</property>

<property>
  <name>dfs.safemode.threshold.pct</name>
  <value>1</value>
  <description>
     testtesttest
</description>
</property>

</configuration>


[root@node01 ~]# vi /opt/hadoop/etc/hadoop/mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<!-- 指定mr框架为yarn方式 -->
<property>
  <name>mapreduce.framework.name</name>
  <value>yarn</value>
</property>
<property>
  <name>mapreduce.application.classpath</name>
  <value>/opt/hadoop/share/hadoop/mapreduce/*,/opt/hadoop/share/hadoop/mapreduce/lib/*</value>
</property>
</configuration>

[root@node01 ~]# vi /opt/hadoop/etc/hadoop/yarn-site.xml 
<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<property>
<!-- 启用RM HA -->
  <name>yarn.resourcemanager.ha.enabled</name>
  <value>true</value>
</property>
<property>
<!-- RM集群标识 -->
  <name>yarn.resourcemanager.cluster-id</name>
  <value>cluster1</value>
</property>
<property>
<!-- RM的逻辑ID列表 -->
  <name>yarn.resourcemanager.ha.rm-ids</name>
  <value>rm1,rm2</value>
</property>
<property>
<!-- RM1的服务地址 -->
  <name>yarn.resourcemanager.hostname.rm1</name>
  <value>node01</value>
</property>
<property>
<!-- RM2的服务地址 -->
  <name>yarn.resourcemanager.hostname.rm2</name>
  <value>node02</value>
</property>

<property>
<!-- ZooKeeper集群的地址 -->
  <name>yarn.resourcemanager.zk-address</name>
  <value>node01:2181,node02:2181,node03:2181</value>
</property>
<property>
<!--配置NodeManager上运行的附属服务。需要配置成mapreduce_shuffle后才可以在Yarn上运行MapReduce程序。-->
  <name>yarn.nodemanager.aux-services</name>
  <value>mapreduce_shuffle</value>
</property>
</configuration>

2.分发到node02，node03节点

[root@node01 ~]# scp -r /opt/hadoop   node02:/opt
[root@node01 ~]# scp -r /opt/hadoop   node03:/opt
#加载环境变量
[root@node01 ~]# vi /root/.bashrc
export JAVA_HOME=/usr/local/jdk
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export HADOOP_HOME=/opt/hadoop
export PATH=$PATH:$HOME/.local/bin:$HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
[root@node02 ~]# vi /root/.bashrc
export JAVA_HOME=/usr/local/jdk
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export HADOOP_HOME=/opt/hadoop
export PATH=$PATH:$HOME/.local/bin:$HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
[root@node03 ~]# vi /root/.bashrc
export JAVA_HOME=/usr/local/jdk
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export HADOOP_HOME=/opt/hadoop
export PATH=$PATH:$HOME/.local/bin:$HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

[root@node03 ~]# source /root/.bashrc
[root@node03 ~]# source /root/.bashrc
[root@node03 ~]# source /root/.bashrc

3.初始化集群

#启动journalnode服务
[root@node01 ~]# hadoop-daemon.sh  start journalnode
[root@node02 ~]# hadoop-daemon.sh  start journalnode
[root@node03 ~]# hadoop-daemon.sh  start journalnode

#格式化namenode
[root@node01 ~]# hdfs namenode -format
#启动namenode
[root@node01 ~]# hadoop-daemon.sh  start namenode

#副namenode节点同步数据
[root@node02 ~]# hdfs namenode -bootstrapStandby
#启动副namenode节点
[root@node02 ~]# hadoop-daemon.sh  start namenode
#任意节点执行，注册zookeeper信息
[root@node01 ~]# hdfs zkfc -formatZK
#关闭所有服务
[root@node01 ~]# stop-dfs.sh
#重新启动hdfs启动
[root@node01 ~]# start-all.sh

5.检测集群启动正常

#查看是否启动7个服务组件
[root@node01 ~]# jps
20081 JournalNode
20645 ResourceManager
20806 NodeManager
20265 DFSZKFailoverController
4123 QuorumPeerMain
105149 Jps
19837 DataNode
22750 NameNode

[root@node02 ~]# jps
130305 NameNode
11089 DataNode
11171 JournalNode
11332 ResourceManager
4647 QuorumPeerMain
101097 Jps
11419 NodeManager
11245 DFSZKFailoverController

#node03仅为数据冗余节点
[root@node03 ~]# jps
8592 JournalNode
8513 DataNode
6113 Jps
8674 NodeManager
4123 QuorumPeerMain

访问web管理页面

http://172.21.36.35:9870/

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-0xDCTRCS-1601190302647)(Hadoop hdfs HA集群部署.assets/image-20200927144234155.png)]

http://172.21.36.36:9870/

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-3D1ewqz1-1601190302652)(Hadoop hdfs HA集群部署.assets/image-20200927144303810.png)]

9837 DataNode
22750 NameNode

[root@node02 ~]# jps
130305 NameNode
11089 DataNode
11171 JournalNode
11332 ResourceManager
4647 QuorumPeerMain
101097 Jps
11419 NodeManager
11245 DFSZKFailoverController

#node03仅为数据冗余节点
[root@node03 ~]# jps
8592 JournalNode
8513 DataNode
6113 Jps
8674 NodeManager
4123 QuorumPeerMain


##### 访问web管理页面
http://172.21.36.35:9870/
http://172.21.36.36:9870/