Hadoop大数据开发__YARN分布式集群安装部署

1、在hadoop01节点上修改配置文件

cd /home/hadoop/app/hadoop/etc/hadoop

修改mapred-site.xml

<configuration>
	<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>
</configuration>
<!-- 指定运行MapReduce的环境是YARN,与hadoop1不同的地方 -->

修改yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
	<property>
		<name>yarn.resourcemanager.connect.retry-interval.ms</name>
		<value>2000</value>
	</property>
	<!-- 超时的周期 -->
	<property>
		<name>yarn.resourcemanager.ha.enabled</name>
		<value>true</value>
	</property>
	<!-- 打开高可用 -->
	<property>
		<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
		<value>true</value>
	</property>
	<!-- 启动故障自动恢复 -->
	<property>
		<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
		<value>true</value>
	</property>
	<property>
		<name>yarn.resourcemanager.cluster-id</name>
		<value>yarn-rm-cluster</value>
	</property>
	<!-- 给YARN Cluster起个名字yarn-rm-cluster -->
	<property>
		<name>yarn.resourcemanager.ha.rm-ids</name>
		<value>rm1,rm2</value>
	</property>
	<!-- 给ResourceManager起个名字rm1、rm2 -->
	<property>
		<name>yarn.resourcemanager.hostname.rm1</name>
		<value>hadoop01</value>
	</property>
	<!-- 配置ResourceManager rm1 hostname -->
	<property>
		<name>yarn.resourcemanager.hostname.rm2</name>
		<value>hadoop02</value>
	</property>
	<!-- 配置ResourceManager rm2 hostname -->
	<property>
		<name>yarn.resourcemanager.recovery.enabled</name>
		<value>true</value>
	</property>
	<!-- 启动ResourceManager自动恢复 -->
	<property>
		<name>yarn.resourcemanager.zk.state-store.address</name>
		<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
	</property>
	<!-- 配置Zookeeper地址 -->
	<property>
		<name>yarn.resourcemanager.zk-address</name>
		<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
	</property>
	<!-- 配置Zookeeper地址 -->
	<property>
		<name>yarn.resourcemanager.address.rm1</name>
		<value>hadoop01:8032</value>
	</property>
	<!-- rm1端口 -->
	<property>
		<name>yarn.resourcemanager.scheduler.address.rm1</name>
		<value>hadoop01:8034</value>
	</property>
	<!-- rm1调度器端口 -->
	<property>
		<name>yarn.resourcemanager.webapp.address.rm1</name>
		<value>hadoop01:8088</value>
	</property>
	<!-- rm1 webapp端口 -->
	<property>
		<name>yarn.resourcemanager.address.rm2</name>
		<value>hadoop02:8032</value>
	</property>
	<!-- rm2端口 -->
	<property>
		<name>yarn.resourcemanager.scheduler.address.rm2</name>
		<value>hadoop02:8034</value>
	</property>
	<!-- rm2调度器端口 -->
	<property>
		<name>yarn.resourcemanager.webapp.address.rm2</name>
		<value>hadoop02:8088</value>
	</property>
	<!-- rm2 webapp端口 -->
	<property>
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
	</property>
	<!-- 执行MapReduce需要配置的shuffle过程 -->
	<property>
		<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
		<value>org.apache.hadoop.mapred.ShuffleHandler</value>
	</property>
	<property>
		<name>mapreduce.jobhistory.address</name>
		<value>hadoop01:10020</value>
	</property>
	<property>
		<name>mapreduce.jobhistory.webapp.address</name>
		<value>hadoop01:19888</value>
	</property>
	<property>
		<name>yarn.application.classpath</name>
		<value>/home/hadoop/app/hadoop-3.3.1/etc/hadoop:/home/hadoop/app/hadoop-3.3.1/share/hadoop/common/lib/*:/home/hadoop/app/hadoop-3.3.1/share/hadoop/common/*:/home/hadoop/app/hadoop-3.3.1/share/hadoop/hdfs:/home/hadoop/app/hadoop-3.3.1/share/hadoop/hdfs/lib/*:/home/hadoop/app/hadoop-3.3.1/share/hadoop/hdfs/*:/home/hadoop/app/hadoop-3.3.1/share/hadoop/mapreduce/*:/home/hadoop/app/hadoop-3.3.1/share/hadoop/yarn:/home/hadoop/app/hadoop-3.3.1/share/hadoop/yarn/lib/*:/home/hadoop/app/hadoop-3.3.1/share/hadoop/yarn/*</value>
	</property>
	<!-- 配置CLASSPATH -->
</configuration>

2、向所有节点hadoop02、hadoop03同步YARN配置文件

cd /home/hadoop/app/hadoop/etc/hadoop
#同步mapred-site.xml
scp -r mapred-site.xml hadoop@hadoop02:/home/hadoop/app/hadoop/etc/hadoop
scp -r mapred-site.xml hadoop@hadoop03:/home/hadoop/app/hadoop/etc/hadoop
#同步yarn-site.xml
scp -r yarn-site.xml hadoop@hadoop02:/home/hadoop/app/hadoop/etc/hadoop
scp -r yarn-site.xml hadoop@hadoop03:/home/hadoop/app/hadoop/etc/hadoop

3、启动YARN集群

#在hadoop01上，一键启动YARN集群
cd /home/hadoop/app/hadoop/sbin
start-yarn.sh
#查看状态，一个是active，一个是standby
cd /home/hadoop/app/hadoop/bin
yarn rmadmin -getServiceState rm1
yarn rmadmin -getServiceState rm2
#打开浏览器进行验证
http://hadoop01:8088

4、命令行验证（利用hadoop自带的mapreduce程序）

cd /home/hadoop/app/hadoop/bin
hadoop jar /home/hadoop/app/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.1.jar wordcount /test/djt.txt /test/out/

在http://hadoop01:8088上能够看到执行状态是SUCCEEDED。