Hadoop 完全分布式集群搭建配置

1 环境

java 必须安装

三台机器ssh免密登录

2 master机器配置

    2.1 core-site.xml    

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:8020</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/abc/hadoop/data/tmp</value>
    </property>
</configuration>

2.2 hdfs-site.xml

<configuration>
     <!-- 指定HDFS副本的数量 -->
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
    <property>
        <!--指定secondaryNameNode的地址和端口号-->
        <name>dfs.namenode.secondary.http-address</name>
        <value>node1:50090</value>
    </property>
   <property>
        <name>dfs.name.dir</name>
        <value>/home/abc/hadoop/hdfs/name</value>
    </property>
    <property>
        <name>dfs.data.dir</name>
        <value>/home/abc/hadoop/hdfs/data</value>
    </property>

</configuration>

2.3  hadoop-env.sh

末尾添加

export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

2.4  mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <!--historyserver地址为server3-->
        <name>mapreduce.jobhistory.address</name>
        <value>node2:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>node2:19888</value>
    </property>
</configuration>

2.5 yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>

    <property>
        <!--resourcemanager服务指定地址-->
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
    </property>

    <property>
        <!--resourcemanager服务指定地址-->
        <name>yarn.application.classpath</name>
        <value>
           /home/abc/hadoop/etc/hadoop:/home/abc/hadoop/share/hadoop/common/lib/*:/home/abc/hadoop/share/hadoop/common/*:
           /home/abc/hadoop/share/hadoop/hdfs:/home/abc/hadoop/share/hadoop/hdfs/lib/*:/home/abc/hadoop/share/hadoop/hdfs/*:
/home/abc/hadoop/share/hadoop/mapreduce/lib/*:/home/abc/hadoop/share/hadoop/mapreduce/*:/home/abc/hadoop/share/hadoop/yarn:
           /home/abc/hadoop/share/hadoop/yarn/lib/*:/home/abc/hadoop/share/hadoop/yarn/*
        </value>
    </property>

  <property>
        <!--启动日志聚集功能-->
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
    <property>
        <!--日志保存时间-->
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>106800</value>
    </property>
    <property>
         <name>yarn.log.server.url</name>
         <value>http://node2:19888/jobhistory/logs</value>
    </property>
</configuration>

 

2.6 slaves

master
node1
node2

3  master机器上， 

    scp /home/abc/hadoop/* node1:/home/abc/hadooop

    scp /home/abc/hadoop/* node2:/home/abc/hadooop

4  三台机器sbin目录下，分别执行./start-all.sh