1. 在/opt下新建ha目录,并将hadoop安装包拷贝一份
sudo mkdir /opt/ha
sudo chown atguigu:atguigu /opt/ha
cp -r /opt/module/hadoop-3.1.3 /opt/ha
rm -rf /opt/ha/hadoop-3.1.3/data /opt/ha/hadoop-3.1.3/logs
2. 修改/opt/ha/hadoop-3.1.3的配置文件
- core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/ha/hadoop-3.1.3/data</value>
</property>
<property>
<name>hadoop.proxyuser.atguigu.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.atguigu.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.http.staticuser.user</name>
<value>atguigu</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>hadoop102:2181,hadoop103:2181,hadoop104:2181</value>
</property>
</configuration>
- hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>dfs.nameservices</name>
<value>mycluster</value>
</property>
<property>
<name>dfs.ha.namenodes.mycluster</name>
<value>nn1,nn2, nn3</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn1</name>
<value>hadoop102:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn2</name>
<value>hadoop103:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn3</name>
<value>hadoop104:8020</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn1</name>
<value>hadoop102:9870</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn2</name>
<value>hadoop103:9870</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn3</name>
<value>hadoop104:9870</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>${hadoop.tmp.dir}/jn</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hadoop102:8485;hadoop103:8485;hadoop104:8485/mycluster</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.mycluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/atguigu/.ssh/id_ecdsa</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
</configuration>
3. 发送hadoop到所有 节点
sudo xsync /opt/ha
4. 修改环境变量
#HADOOP_HOME
export HADOOP_HOME=/opt/ha/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
sudo xsync /etc/profile.d/my_env.sh
source一下或者重启session
5. 启动JournalNode
在任意一台节点执行
hdfs --workers --daemon start journalnode
6. 102格式化NN
在102上执行格式化命令,并启动Namenode
hdfs namenode -format
hdfs --daemon start namenode
7. 103,104同步102的元数据
在103,104上执行同步命令,并启动Namenode
hdfs namenode -bootstrapStandby
hdfs --daemon start namenode
8. 初始化zkfc在zookeeper中的节点
先启动zookeeper集群后,在任意一台节点格式化zkfc
hdfs zkfc -formatZK
9. 启动zkfc
在102,103,104上分别启动zkfc
hdfs --daemon start zkfc
10. 启动datanodes
在任意一台节点执行
hdfs --workers --daemon start datanode
11. 假设出了问题,想重新来,怎么办
- 关闭集群,在所有节点执行
killall -9 java
- 在所有节点删除data logs文件夹
rm -rf /opt/ha/hadoop-3.1.3/data /opt/ha/hadoop-3.1.3/logs
- 启动Zookeeper集群
- 执行第5,6,7,9,10步