Hadoop 2.6.4 分布式搭建

多集群安装(3个节点)
节点详情：

vm01---192.168.110.101(master)
vm02---192.168.110.102(SecondNamenode)
vm03---192.168.110.103
在slave里配置SecondNamenode

master:
192.168.110.101
slaves:
vm02:192.168.110.102
vm03:192.168.110.103

core-site.xml

<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://192.168.110.101:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/opt/hadoop-2.6.4/tmp</value>
        </property>
        <property>
                <name>fs.checkpoint.period</name>
                <value>3600</value>
        </property>
        <property>
                <name>fs.checkpoint.size</name>
                <value>67108864</value>
        </property>
</configuration>

hdfs-site.xml

<configuration>
        <property>
                <name>dfs.replication</name>
                <value>3</value>
        </property>
        <property>
                <name>dfs.http.address</name>
                <value>vm01:50070</value>
        </property>
        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>vm02:50090</value>
        </property>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>/opt/hadoop-2.6.4/name</value>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/opt/hadoop-2.6.4/data</value>
        </property>
<property>
                <name>dfs.permissions</name>
                <value>false</value>
        </property>
</configuration>

yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
        <property>
                <name>yarn.resourcemanager.hostname</name>
                <value>vm01</value>
        </property>
        <!-- 是否启用日志聚集功能 -->
        <property>
                <name>yarn.log-aggregation-enable</name>
                <value>true</value>
        </property>
        <!-- 日志保留时间(单位为秒) -->
        <property>
                <name>yarn.log-aggregation.retain-seconds</name>
                <value>106800</value>
        </property>
        <property>
                <name>yarn.resourcemanager.address</name>
                <value>vm01:8032</value>
         </property>
        <property>
                <name>yarn.resourcemanager.scheduler.address</name>
                <value>vm01:8030</value>
        </property>
        <property>
                <name>yarn.resourcemanager.resource-tracker.address</name>
                <value>vm01:8031</value>
        </property>
        <!-- mr运行不验证虚拟内存    -->
        <property>
                <name>yarn.nodemanager.vmem-check-enabled</name>
                <value>false</value>
                <description>Whether virtual memory limits will be enforced for containers</description>
        </property>
        <!--修改虚拟内存倍数，默认是2.1    -->
        <property>
                <name>yarn.nodemanager.vmem-pmem-ratio</name>
                <value>4</value>
                <description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
        </property>

</configuration>

mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <!--
    <property>
        <name>mapred.map.child.java.opts</name>
            <value>-Xmx1024m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8883</value>
    </property>
    <property>
        <name>mapred.reduce.child.java.opts</name>
        <value>-Xmx1024m -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8884</value>
     </property>
     -->
     <property>
        <name>mapred.tasktracker.map.tasks.maximum</name>
        <value>1</value>
    </property>
     <property>
        <name>mapred.tasktracker.reduce.tasks.maximum</name>
        <value>1</value>
     </property>

</configuration>

遇到一个严重的问题，虚拟内存溢出，一开始以为是开的虚拟机内存不足，后来加了内存后，还是存在这个问题

后来在mapred-site.xml注释掉与JVM运行内存空间的相关的配置就可以了，具体原理有待研究。