hadoop的yarn部署

最新推荐文章于 2024-07-17 14:54:09 发布

zhoucanji

最新推荐文章于 2024-07-17 14:54:09 发布

阅读量301

点赞数 1

文章标签： hadoop eclipse linux

本文链接：https://blog.csdn.net/zhoucanji/article/details/139397788

版权

MapReduce配置文件

mapred-env.sh,新增环境变量

[root@hadoop1 hadoop]# vi /export/server/hadoop/etc/hadoop/mapred-env.sh
export JAVA_HOME=/export/server/jdk
export HAPOOD_JOB_HISTORYSERVER_HEAPSIZE=1000
export HAPOOD_MAPRED_ROOT_LOGGER=INFO,RFA

mapred-site.xml,新增环境变量

[root@hadoop1 hadoop]# vi /export/server/hadoop/etc/hadoop/mapred-site.xml
<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
                <description>运行框架设置</description>
        </property>
        <property>
                <name>mapreduce.jobhistory.address</name>
                <value>hadoop1:10020</value>
                <description>历史服务器通讯端口</description>
        </property>
        <property>
                <name>mapreduce.jobhistory.webapp.address</name>
                <value>hadoop1:19888</value>
                <description>历史服务器web地址</description>
        </property>
        <property>
                <name>mapreduce.jobhistory.intermediate-done-dir</name>
                <value>/data/mspReduceHistory/temp</value>
                <description>历史信息在hdfs的记录路径</description>
        </property>
        <property>
                <name>yarn.app.mapreduce.am.env</name>
                <value>/data/mspReduceHistory/done</value>
                <description>历史信息在hdfs的记录路径</description>
        </property>
        <property>
                <name>yarn.app.mapreduce.am.env</name>
                <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
                <description>MapReduce Home 设置为 HADOOP_HOME</description>
        </property>
        <property>
                <name>mapreduce.map.env</name>
                <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
                <description>MapReduce Home 设置为 HADOOP_HOME</description>
        </property>
        <property>
                <name>mapreduce.reduce.env</name>
                <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
                <description>MapReduce Home 设置为 HADOOP_HOME</description>
        </property>
</configuration>

yarn配置文件

yarn-env.sh

[root@hadoop1 hadoop]# mkdir -p /data/hadoopLog
[root@hadoop1 hadoop]# vi /export/server/hadoop/etc/hadoop/yarn-env.sh
export JAVA_HOME=/export/server/jdk
export HADOOP_HOME=/export/server/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_LOG_DIR=/data/hadoopLog

yarn-site.xml

[root@hadoop1 data]# mkdir -p /data/nodeManagerLocal
[root@hadoop1 data]# mkdir -p /data/nodeManagerLog
[root@hadoop1 data]# vi /export/server/hadoop/etc/hadoop/yarn-site.xml
<configuration>
	<property>
		<name>yarn.resourcemanager.hostname</name>
		<value>hadoop1</value>
		<description>ResourceManager设置节点</description>
	</property>
	<property>
		<name>yarn.nodemanager.local-dirs</name>
		<value>/data/nodeManagerLocal</value>
		<description>ResourceManager中间数据库本地存储路径</description>
	</property>
	<property>
		<name>yarn.nodemanager.log-dirs</name>
		<value>/data/nodeManagerLog</value>
		<description>ResourceManager数据日志本地存储路径</description>
	</property>
	<property>
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
		<description>为MapReduce程序开启Shuffle服务</description>
	</property>
	<property>
		<name>yarn.log.server.url</name>
		<value>http://hadoop1:19888/jobhistory/log</value>
		<description>历史服务器url</description>
	</property>
	<property>
		<name>yarn.web-proxy.address</name>
		<value>hadoop1:8089</value>
		<description>代理服务器地址</description>
	</property>
	<property>
		<name>yarn.log-aggregation-enable</name>
		<value>true</value>
		<description>开始日志聚合</description>
	</property>
	<property>
		<name>yarn.nodemanager.remote-app-log-dir</name>
		<value>/tmp/logs</value>
		<description>程序日志hdfs的存储路径</description>
	</property>
	<property>
		<name>yarn.resourcemanager.scheduler.class</name>
		<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
		<description>选择公平调度器</description>
	</property>
</configuration>

同步到hadoop2,hadoop3主机

[root@hadoop1 hadoop]# cd /export/server/hadoop/etc/hadoop
[hadoop@hadoop1 hadoop]$ scp mapred-env.sh mapred-site.xml yarn-env.sh yarn-site.xml hadoop2:`pwd`/
[hadoop@hadoop1 hadoop]$ scp mapred-env.sh mapred-site.xml yarn-env.sh yarn-site.xml hadoop3:`pwd`/

集群启动命令

chown -R hadoop:hadoop /data
#一键启动yarn集群
start-yarn.sh
#一键停止yarn集群
stop-yarn.sh
#在当前机器，单独启动和停止进程
yarn --daemon start|stop resourcemanager|nodemanager|proxyserver
#历史服务器启动和停止
mapred --deamon start|stop historyserver