<?xml
version="1.0"?>
<!--
Licensed under the Apache License, Version
2.0 (the "License");
you may not use this file except in
compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed
to in writing, software
distributed under the License is
distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.
See the License for the specific language
governing permissions and
limitations under the License. See accompanying
LICENSE file.-->
<configuration>
<!-- Site
specific YARN configuration properties -->
<!-- reducer获取数据的方式 -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<!-- 指定YARN的ResourceManager的地址 -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>hadoop203.cevent.com</value>
</property>
</configuration>
~
[cevent@hadoop202
hadoop-2.7.2]$ xsync etc/hadoop/yarn-site.xml
fname=yarn-site.xml
pdir=/opt/module/hadoop-HA/hadoop-2.7.2/etc/hadoop
---------------
hadoop203.cevent.com ----------------
sending incremental
file list
yarn-site.xml
sent 1840
bytes received 43 bytes 3766.00 bytes/sec
total size is
2457 speedup is 1.30
---------------
hadoop204.cevent.com ----------------
sending
incremental file list
yarn-site.xml
sent 1840
bytes received 43 bytes 3766.00 bytes/sec
total size is
2457 speedup is 1.30
4.勘察结果hadoop203
<!--指定zookeeper集群的地址-->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>hadoop202.cevent.com:2181,hadoop203.cevent.com:2181,hadoop204.cevent.com:2181</value>
</property>
<!--启用自动恢复-->
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<!--指定resourcemanager的状态信息存储在zookeeper集群-->
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
</configuration>
You have new
mail in /var/spool/mail/root
[cevent@hadoop203
hadoop-2.7.2]$
[cevent@hadoop202
hadoop-2.7.2]$ sbin/hadoop-daemon.sh start
journalnode 启动journalnode
starting
journalnode, logging to
/opt/module/hadoop-HA/hadoop-2.7.2/logs/hadoop-cevent-journalnode-hadoop202.cevent.com.out
You
have new mail in /var/spool/mail/root
[cevent@hadoop202
hadoop-2.7.2]$ bin/hdfs namenode -format 格式化namenode
20/04/23
20:53:20 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG:
Starting NameNode
STARTUP_MSG: host = hadoop202.cevent.com/192.168.1.202
STARTUP_MSG: args = [-format]
STARTUP_MSG:
version = 2.7.2
…….
Re-format filesystem in
Storage Directory /opt/module/hadoop-HA/hadoop-2.7.2/data/tmp/dfs/name ? (Y
or N) y
Re-format filesystem in QJM to
[192.168.1.202:8485, 192.168.1.203:8485, 192.168.1.204:8485] ? (Y or N) y
20/04/23
20:53:36 INFO namenode.FSImage: Allocated new BlockPoolId:
BP-1726989784-192.168.1.202-1587646416928
20/04/23
20:53:36 INFO common.Storage: Storage directory
/opt/module/hadoop-HA/hadoop-2.7.2/data/tmp/dfs/name has been successfully
formatted.
20/04/23
20:53:37 INFO namenode.NNStorageRetentionManager: Going to retain 1 images
with txid >= 0
20/04/23
20:53:37 INFO util.ExitUtil: Exiting with status 0
20/04/23
20:53:37 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down
NameNode at hadoop202.cevent.com/192.168.1.202
************************************************************/[cevent@hadoop202
hadoop-2.7.2]$ sbin/hadoop-daemon.sh start
namenode 启动namenode
starting
namenode, logging to
/opt/module/hadoop-HA/hadoop-2.7.2/logs/hadoop-cevent-namenode-hadoop202.cevent.com.out
[cevent@hadoop202
hadoop-2.7.2]$ jps
3742
Jps
3666
NameNode
3532 JournalNode
6.启动Hadoop203(=resourcemanager2)
[cevent@hadoop203
hadoop-2.7.2]$ bin/hdfs namenode -bootstrapStandby
同步rm1的元数据
20/04/23
20:59:10 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG:
Starting NameNode
STARTUP_MSG: host = hadoop203.cevent.com/192.168.1.203
STARTUP_MSG: args = [-bootstrapStandby]
STARTUP_MSG:
version = 2.7.2
20/04/23
20:59:14 WARN common.Util: Path
/opt/module/hadoop-HA/hadoop-2.7.2/data/tmp/dfs/name should be specified as a
URI in configuration files. Please update hdfs configuration.
=====================================================
About to bootstrap Standby ID
nn2 from:
Nameservice ID: ceventcluster
Other Namenode ID: nn1
Other NN's HTTP address:
http://hadoop202.cevent.com:50070
Other NN's IPC address: hadoop202.cevent.com/192.168.1.202:9000
Namespace ID: 324046010
Block pool ID:
BP-1726989784-192.168.1.202-1587646416928
Cluster ID:
CID-8c5b68c3-24a2-46c1-949b-391664f18d84
Layout version: -63
isUpgradeFinalized: true
=====================================================
Re-format filesystem in
Storage Directory /opt/module/hadoop-HA/hadoop-2.7.2/data/tmp/dfs/name ? (Y
or N) y
20/04/23
20:59:24 INFO common.Storage: Storage directory /opt/module/hadoop-HA/hadoop-2.7.2/data/tmp/dfs/name
has been successfully formatted.
20/04/23
20:59:24 WARN common.Util: Path
/opt/module/hadoop-HA/hadoop-2.7.2/data/tmp/dfs/name should be specified as a
URI in configuration files. Please update hdfs configuration.
20/04/23
20:59:24 WARN common.Util: Path /opt/module/hadoop-HA/hadoop-2.7.2/data/tmp/dfs/name
should be specified as a URI in configuration files. Please update hdfs
configuration.
20/04/23
20:59:26 INFO namenode.TransferFsImage: Opening connection to
http://hadoop202.cevent.com:50070/imagetransfer?getimage=1&txid=0&storageInfo=-63:324046010:0:CID-8c5b68c3-24a2-46c1-949b-391664f18d84
20/04/23
20:59:26 INFO namenode.TransferFsImage: Image Transfer timeout configured to
60000 milliseconds
20/04/23
20:59:26 INFO namenode.TransferFsImage: Transfer took 0.00s at 0.00 KB/s
20/04/23
20:59:26 INFO namenode.TransferFsImage: Downloaded file
fsimage.ckpt_0000000000000000000 size 353 bytes.
20/04/23
20:59:26 INFO util.ExitUtil: Exiting with status 0
20/04/23
20:59:26 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG:
Shutting down NameNode at hadoop203.cevent.com/192.168.1.203
************************************************************/[cevent@hadoop203
hadoop-2.7.2]$ sbin/hadoop-daemon.sh start
namenode 启动namenode
starting namenode, logging to
/opt/module/hadoop-HA/hadoop-2.7.2/logs/hadoop-cevent-namenode-hadoop203.cevent.com.out
7.Hadoop202启动datanode(这里偶遇脑瘫-干掉)
[cevent@hadoop202
hadoop-2.7.2]$ sbin/hadoop-daemons.sh start
datanode 这里的daemeons进程必须
hadoop204.cevent.com:
starting datanode, logging to
/opt/module/hadoop-HA/hadoop-2.7.2/logs/hadoop-cevent-datanode-hadoop204.cevent.com.out
hadoop203.cevent.com:
starting datanode, logging to /opt/module/hadoop-HA/hadoop-2.7.2/logs/hadoop-cevent-datanode-hadoop203.cevent.com.out
hadoop202.cevent.com:
starting datanode, logging to
/opt/module/hadoop-HA/hadoop-2.7.2/logs/hadoop-cevent-datanode-hadoop202.cevent.com.out
hadoop205.cevent.com:
ssh: connect to host hadoop205.cevent.com port 22: No route to host
[cevent@hadoop202
hadoop-2.7.2]$ jps
3936 DataNode
3666 NameNode
3532 JournalNode
4009 Jps
【偶遇脑瘫split-brain】
[cevent@hadoop202
hadoop-2.7.2]$ bin/hdfs haadmin
-transitionToActive nn1 启动hadoop202的active
Automatic
failover is enabled for NameNode at hadoop203.cevent.com/192.168.1.203:9000
Refusing
to manually manage HA state, since it may cause
a split-brain scenario or
other incorrect state.If you are very sure you know
what you are doing, please
specify the --forcemanual flag. 要求必须手动标识
[cevent@hadoop202
hadoop-2.7.2]$ jps
4564
Jps
3666
NameNode
3532 JournalNode
[cevent@hadoop202
hadoop-2.7.2]$ bin/hdfs haadmin
-transitionToActive --forcemanual nn1 执行激活
You
have specified the --forcemanual flag. This flag is dangerous, as it can
induce a split-brain scenario that WILL CORRUPT your HDFS namespace, possibly
irrecoverably.
It is
recommended not to use this flag, but instead to shut down the cluster and
disable automatic failover if you prefer to manually manage your HA state.
You
may abort safely by answering 'n' or hitting ^C now.
Are you sure you want to continue?
(Y or N) y
20/04/23
21:24:11 WARN ha.HAAdmin: Proceeding with manual HA state management even
though
automatic
failover is enabled for NameNode at hadoop203.cevent.com/192.168.1.203:9000
20/04/23
21:24:11 WARN ha.HAAdmin: Proceeding with manual HA state management even
though
automatic
failover is enabled for NameNode at hadoop202.cevent.com/192.168.1.202:9000
[cevent@hadoop202
hadoop-2.7.2]$ jps
4691
Jps
3666
NameNode
3532 JournalNode
1.原版yarn-site.xml <?xml version="1.0"?> <!-- Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the ...