第二十篇:YARN安装

1.修改配置文件

1.1.yarn-site.xml配置

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>
<property>
	<name>yarn.resourcemanager.connect.retry-interval.ms</name>
	<value>2000</value>
</property>
<property>
	<name>yarn.resourcemanager.ha.enabled</name>
	<value>true</value>
</property>
<!--打开高可用-->
<property>
	<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
	<value>true</value>
</property>
<!--启动故障自动恢复-->
<property>
	<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
	<value>true</value>
</property>
<!--rm启动内置选举active-->
<property>
	<name>yarn.resourcemanager.cluster-id</name>
	<value>yarn-rm-cluster</value>
</property>
<!--给yarn cluster 取个名字yarn-rm-cluster-->
<property>
	<name>yarn.resourcemanager.ha.rm-ids</name>
	<value>rm1,rm2</value>
</property>
<!--ResourceManager高可用 rm1,rm2-->
<property>
	<name>yarn.resourcemanager.hostname.rm1</name>
	<value>hadoop01</value>
</property>
<property>
	<name>yarn.resourcemanager.hostname.rm2</name>
	<value>hadoop02</value>
</property>
<property>
	<name>yarn.resourcemanager.recovery.enabled</name>
	<value>true</value>
</property>
<!--启用resourcemanager 自动恢复-->
<property>
	<name>yarn.resourcemanager.zk.state-store.address</name>
	<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
</property>
<!--状态存储地址-->
<property>
	<name>yarn.resourcemanager.zk-address</name>
	<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
</property>
<!--配置Zookeeper地址-->
<property>
	<name>yarn.resourcemanager.address.rm1</name>
	<value>hadoop01:8032</value>
</property>
<!--rm1端口号-->
<property>
	<name>yarn.resourcemanager.scheduler.address.rm1</name>
	<value>hadoop01:8034</value>
</property>
<!-- rm1调度器的端口号-->
<property>
	<name>yarn.resourcemanager.webapp.address.rm1</name>
	<value>hadoop01:8088</value>
</property>
<!-- rm1 webapp端口号-->
<property>
	<name>yarn.resourcemanager.address.rm2</name>
	<value>hadoop02:8032</value>
</property>
<property>
	<name>yarn.resourcemanager.scheduler.address.rm2</name>
	<value>hadoop02:8034</value>
</property>
<property>
	<name>yarn.resourcemanager.webapp.address.rm2</name>
	<value>hadoop02:8088</value>
</property>
<property>
	<name>yarn.nodemanager.aux-services</name>
	<value>mapreduce_shuffle</value>
</property>
<property>
	<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
	<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<!--执行MapReduce需要配置的shuffle过程-->
</configuration>

1.2.mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
	<name>mapreduce.framework.name</name>
	<value>yarn</value>
</property>
<!--MapReduce以yarn模式运行-->
</configuration>

2.配置文件同步到其他节点

[hadoop@hadoop01 hadoop]$ deploy.sh mapred-site.xml /home/hadoop/app/hadoop/etc/hadoop slave
mapred-site.xml                                                                                                100%  876     0.9KB/s   00:00    
mapred-site.xml                                                                                                100%  876     0.9KB/s   00:00  
[hadoop@hadoop01 hadoop]$ deploy.sh yarn-site.xml /home/hadoop/app/hadoop/etc/hadoop slave
yarn-site.xml                                                                                                  100% 3089     3.0KB/s   00:00    
yarn-site.xml                                                                                                  100% 3089     3.0KB/s   00:00 

3.一键启动YARN

[hadoop@hadoop01 hadoop]$ sbin/start-yarn.sh 
starting yarn daemons
starting resourcemanager, logging to /home/hadoop/app/hadoop-2.6.0-cdh5.10.0/logs/yarn-hadoop-resourcemanager-hadoop01.out
hadoop03: starting nodemanager, logging to /home/hadoop/app/hadoop-2.6.0-cdh5.10.0/logs/yarn-hadoop-nodemanager-hadoop03.out
hadoop02: starting nodemanager, logging to /home/hadoop/app/hadoop-2.6.0-cdh5.10.0/logs/yarn-hadoop-nodemanager-hadoop02.out
hadoop01: starting nodemanager, logging to /home/hadoop/app/hadoop-2.6.0-cdh5.10.0/logs/yarn-hadoop-nodemanager-hadoop01.out

4.启动备用节点RM

[hadoop@hadoop02 hadoop]$ sbin/yarn-daemon.sh start resourcemanager
starting resourcemanager, logging to /home/hadoop/app/hadoop-2.6.0-cdh5.10.0/logs/yarn-hadoop-resourcemanager-hadoop02.out

5.查看RM状态

[hadoop@hadoop01 hadoop]$ bin/yarn rmadmin -getServiceState rm1
20/08/14 01:03:01 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
active
[hadoop@hadoop01 hadoop]$ bin/yarn rmadmin -getServiceState rm2
20/08/14 01:03:15 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
standby
[hadoop@hadoop01 hadoop]$ 

6.web界面查看YARN

http://master:8088/cluster

7.测试运行YARN

  运行 Hadoop 自带 Wordcount 程序

[hadoop@hadoop01 hadoop]$ bin/hadoop jar share/hadoop/mapreduce2/hadoop-mapreduce-examples-2.6.0-cdh5.10.0.jar wordcount /test/wd.txt /test/output

bin/hadoop jar share/hadoop/mapreduce2/hadoop-mapreduce-examples-2.6.0-cdh5.10.0.jar wordcount /test/wd.txt /test/output
查看执行结果

[hadoop@hadoop01 hadoop]$ bin/hdfs dfs -cat /test/output/*

©️2020 CSDN 皮肤主题: 大白 设计师:CSDN官方博客 返回首页