yarn参数调优-充分利用服务器资源

HDP推荐: https://docs.cloudera.com/HDPDocuments/HDP2/HDP-2.1.1/bk_installing_manually_book/content/rpm-chap1-11.html

我的例子: 64H 64G 2DISKS
CORES为机器CPU核数
DISKS为机器上挂载的磁盘个数
Total available RAM为机器总内存
MIN_CONTAINER_SIZE是指container最小的容量大小，这需要根据具体情况去设置，可以参考下面的表格

每台机器的可用的RAM	container最小值
小余4GB	256MB
4GB到8GB	512MB
8GB到24GB	1024MB
大于24GB	2GB

containers = min (2CORES, 1.8DISKS, (Total available RAM) / MIN_CONTAINER_SIZE) = min(264, 1.82, 64/2) = 3.6 = 4
RAM-per-container = max(MIN_CONTAINER_SIZE, (Total Available RAM) / containers)) = max(2, 64/4) = 16

yarn-site.xml yarn.nodemanager.resource.memory-mb 8192 MB = containers * RAM-per-container = 4 * 16 = 64 = 65536
yarn-site.xml yarn.scheduler.minimum-allocation-mb 1024MB = RAM-per-container = 16 = 16384
yarn-site.xml yarn.scheduler.maximum-allocation-mb 8192 MB = containers * RAM-per-container = 4 * 16 = 64 = 65536
yarn-site.xml (check) yarn.app.mapreduce.am.resource.mb 1536 MB = 2 * RAM-per-container = 2 * 16 = 32 = 32768
yarn-site.xml (check) yarn.app.mapreduce.am.command-opts -Xmx1024m = 0.8 * 2 * RAM-per-container = 0.8 * 2 * 16 = 25 = 25600
mapred-site.xml mapreduce.map.memory.mb 1024 MB = RAM-per-container = 16 = 16384
mapred-site.xml mapreduce.reduce.memory.mb 1024 MB = 2 * RAM-per-container = 32 = 32768
mapred-site.xml mapreduce.map.java.opts = 0.8 * RAM-per-container = 0.8 * 16 = 12.8 = 13107
mapred-site.xml mapreduce.reduce.java.opts = 0.8 * 2 * RAM-per-container = 0.8 * 2 * 16 = 25 = 25600

<!-- yarn-site.xml -->
<property>
	<name>yarn.nodemanager.resource.memory-mb</name>
	<value>65536</value>
</property>
<property>
	<name>yarn.scheduler.minimum-allocation-mb</name>
	<value>16384</value>
</property>
<property>
	<name>yarn.scheduler.maximum-allocation-mb</name>
	<value>65536</value>
</property>
<property>
	<name>yarn.app.mapreduce.am.resource.mb</name>
	<value>32768</value>
</property>
<property>
	<name>yarn.app.mapreduce.am.command-opts</name>
	<value>-Xmx25600m</value>
</property>
<property>
	<name>yarn.nodemanager.resource.cpu-vcores</name>
	<value>63</value>
</property>
<property>
	<name>yarn.scheduler.maximum-allocation-vcores</name>
	<value>441</value>
</property>

<!-- mapred-site.xml -->
<property>
	<name>mapreduce.map.memory.mb</name>
	<value>16384</value>
</property>
<property>
	<name>mapreduce.reduce.memory.mb</name>
	<value>32768</value>
</property>
<property>
	<name>mapreduce.map.java.opts</name>
	<value>-Xmx13107m</value>
</property>
<property>
	<name>mapreduce.reduce.java.opts</name>
	<value>-Xmx25600m</value>
</property>