hadoop2.6.4分布式配置安装

操作系统 ubuntu 14.04 64bit

软件  hadoop-2.6.4.tar.gz   jdk-8u91-linux-x64.tar.gz

电脑3台   

         master        mater.hadoop 192.168.1.84

          slave1       s91.hadoop 192.168.1.91

           slave2       s131.hadoop 192.168.1.131

准备工作: 编辑 /etc/hosts 和 /etc/hostname修改成上述指定

1.  解压 jdk和hadoop

    tar xzvf jdk-8u91-linux-x64.tar.gz

   tar xzvf hadoop-2.6.4.tar.gz

2. 编辑 /etc/profile 增加如下内容

export JAVA_HOME=/home/hadoop/bigdata/jdk1.8.0_91
export HADOOP_HOME=/home/hadoop/bigdata/hadoop-2.6.6
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib

export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

3.修改hadoop配置文件 ../hadoop-2.6.4/etc/hadoop/

a. 修改hadoop-env.sh  与 yarn-env.sh

    增加 export JAVA_HOME=/home/hadoop/bigdata/jdk1.8.0_91

b. 修改slaves文件

   增加 s91.hadoop
            s131.hadoop

c. 修改hdfs-site.xml

  <configuration>
<property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>master.hadoop:9001</value>
</property>
<property>
    <name>dfs.blocksize</name>
    <value>268435456</value>
</property>
<property>
    <name>dfs.namenode.handler.count</name>
    <value>100</value>
</property>
<property>
    <name>dfs.namenode.name.dir</name>
    <value>/home/hadoop/bigdata/hdfs/name</value>
</property>
<property>
    <name>dfs.datanode.data.dir</name>
    <value>/home/hadoop/bigdata/hdfs/data</value>
</property>
<property>
    <name>dfs.replication</name>
    <value>2</value>
</property>
<property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
</property>

</configuration>

d.修改core-site.xml

<configuration>
<property>
    <name>fs.defaultFS</name>
    <value>hdfs://master.hadoop:9000</value>
</property>
<property>
    <name>io.file.buffer.size</name>
    <value>131072</value>
</property>
<property>
    <name>hadoop.tmp.dir</name>
    <value>/home/hadoop/bigdata/tmp</value>
</property>

</configuration>

e.修改mapred-site.xml

<configuration>
<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>
<property>
    <name>mapreduce.jobhistroy.address</name>
    <value>master.hadoop:10020</value>
</property>
<property>
    <name>mapreduce.jobhistroy.webapp.address</name>
    <value>master.hadoop:19888</value>
</property>
</configuration>

f.修改yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
<!--property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property-->
<property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>8192</value>
</property>
<property>
    <name>yarn.resourcemanager.address</name>
    <value>master.hadoop:8032</value>
</property>
<property>
    <name>yarn.resourcemanager.scheduler.address</name>
    <value>master.hadoop:8030</value>
</property>
<property>
    <name>yarn.resourcemanager.resource-tracker.address</name>
    <value>master.hadoop:8035</value>
</property>
<property>
    <name>yarn.resourcemanager.admin.address</name>
    <value>master.hadoop:8033</value>
</property>
<property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>master.hadoop:8088</value>
</property>

</configuration>

4.拷贝配置文件到slave机器

scp -r ../etc/hadoop/* hadoop@s91.hadoop:/home/hadoop/bigdata/hadoop2.6.4/etc/hadoop/

scp -r ../etc/hadoop/* hadoop@s131.hadoop:/home/hadoop/bigdata/hadoop2.6.4/etc/hadoop/

5.启动dfs和yarn

格式化namenode和datanode

../bin/hdfs namenode -format

../bin/hdfs datanode -format

./start-dfs.sh

master上jps可以看到
8840 NameNode
9149 SecondaryNameNode
11311 Jps

slave上jps可以看到
7002 Jps
5643 DataNode

./start-yarn.sh

master上jps可以看到

10866 ResourceManager
8840 NameNode
9149 SecondaryNameNode
11311 Jps
slave上jps可以看到

6793 NodeManager
7002 Jps
5643 DataNode

停止命令 ./stop-yarn.sh   ./stop-dfs.sh

6. 网页查看

http://master.hadoop:50070

http://master.hadoop:8088