我的例子: 64H 64G 2DISKS
CORES为机器CPU核数
DISKS为机器上挂载的磁盘个数
Total available RAM为机器总内存
MIN_CONTAINER_SIZE是指container最小的容量大小,这需要根据具体情况去设置,可以参考下面的表格
每台机器的可用的RAM | container最小值 |
---|---|
小余4GB | 256MB |
4GB到8GB | 512MB |
8GB到24GB | 1024MB |
大于24GB | 2GB |
containers = min (2CORES, 1.8DISKS, (Total available RAM) / MIN_CONTAINER_SIZE) = min(264, 1.82, 64/2) = 3.6 = 4
RAM-per-container = max(MIN_CONTAINER_SIZE, (Total Available RAM) / containers)) = max(2, 64/4) = 16
yarn-site.xml yarn.nodemanager.resource.memory-mb 8192 MB = containers * RAM-per-container = 4 * 16 = 64 = 65536
yarn-site.xml yarn.scheduler.minimum-allocation-mb 1024MB = RAM-per-container = 16 = 16384
yarn-site.xml yarn.scheduler.maximum-allocation-mb 8192 MB = containers * RAM-per-container = 4 * 16 = 64 = 65536
yarn-site.xml (check) yarn.app.mapreduce.am.resource.mb 1536 MB = 2 * RAM-per-container = 2 * 16 = 32 = 32768
yarn-site.xml (check) yarn.app.mapreduce.am.command-opts -Xmx1024m = 0.8 * 2 * RAM-per-container = 0.8 * 2 * 16 = 25 = 25600
mapred-site.xml mapreduce.map.memory.mb 1024 MB = RAM-per-container = 16 = 16384
mapred-site.xml mapreduce.reduce.memory.mb 1024 MB = 2 * RAM-per-container = 32 = 32768
mapred-site.xml mapreduce.map.java.opts = 0.8 * RAM-per-container = 0.8 * 16 = 12.8 = 13107
mapred-site.xml mapreduce.reduce.java.opts = 0.8 * 2 * RAM-per-container = 0.8 * 2 * 16 = 25 = 25600
<!-- yarn-site.xml -->
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>65536</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>16384</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>65536</value>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
<value>32768</value>
</property>
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
<value>-Xmx25600m</value>
</property>
<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>63</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-vcores</name>
<value>441</value>
</property>
<!-- mapred-site.xml -->
<property>
<name>mapreduce.map.memory.mb</name>
<value>16384</value>
</property>
<property>
<name>mapreduce.reduce.memory.mb</name>
<value>32768</value>
</property>
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx13107m</value>
</property>
<property>
<name>mapreduce.reduce.java.opts</name>
<value>-Xmx25600m</value>
</property>