mapreduce的配置及yarn的部署

mapreduce的配置

修改/export/server/etc/hadoop/mapred-env.sh

export JAVA_HOME=/export/server/jdk
#设置jobhistoryserver的进程为1G
export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
#设置日志的等级为info
export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA

修改mapred-site.xml

修改如下：
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
<description>mapreduce的运行框架设置为yarn</description>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>node1:10020</value>
<description>历史服务器的通讯端口为node1:10020</description>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>node1:19888</value>
<description>历史服务器的web端口为node1:19888</description>
</property>
<property>
<name>mapreduce.jobhistory.intermediate-done-dir</name>
<value>/data/mr-history/tmp</value>
<description>历史信息在HDFS的记录临时路径</description>
</property>
<property>
<name>mapreduce.jobhistory.done-dir</name>
<value>yarn</value>
<description>历史信息在HDFS的记录路径</description>
</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
<description>mapreduce_home设置为hadoop_home</description>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
<description>mapreduce_home设置为hadoop_home</description>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
<description>mapreduce_home设置为hadoop_home</description>
</property>

<!--- 可不加
<property>
<name>mapreduce.jobhistory.address</name>
<value>master:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>master:19888</value>
</property>
-->

</configuration>

yarn的配置

修改/export/server/etc/hadoop/yarn-env.sh

export JAVA_HOME=/export/server/jdk
export HADOOP_HOME=/export/server/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_LOG_DIR=$HADOOP_HOME/logs

修改/export/server/etc/hadoop/yarn-site.sh

修改如下：
<property>
<name>yarn.resourcemanager.hostname</name>
<value>node1</value>
<description>resourcemanager设置在node1节点</description>
</property>
<property>
<name>yarn.nodemanager.local-dirs</name>
<value>/data/nm-local</value>
<description>nodemanager中间件的本地linux存储路径</description>
</property>
<property>
<name>yarn.nodemanager.log-dirs</name>
<value>/data/nm-log</value>
<description>nodemanager数据日志本地linux存储路径</description>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
<description>为mapreduce程序开启shuffle服务</description>
</property>

<property>
<name>yarn.log.server.url</name>
<value>http://node1:19888/jobhistory/logs</value>
<description>历史服务器url</description>
</property>
<property>
<name>yarn.web-proxy.address</name>
<value>node1:8089</value>
<description>代理服务器主机和端口</description>
</property>
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
<description>开启日志聚合</description>
</property>
<property>
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/tmp/logs</value>
<description>程序日志HDFS的储存路径</description>
</property>
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
<description>选择公平调度器</description>
</property>


<--可不加
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>master:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>master:8025</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>master:8040</value>
</property>
-->

</configuration>

yarn的启动与关闭

start-yarn.sh
stop-yarn.sh
yarn --daemon start|stop resourcemanager|nodemanager|proxyserver
# 启动和关闭历史服务器
mapred --daemon start|stop historyserver