【Hadoop学习笔记】安装配置

ubuntu 13.10

host1: hadoop01.ko （namenode）       host2: hadoop02.ko（datanode）

Hadoop-2.2.0

ready to go:

目录/datas4hadoop：

新增的磁盘/dev/sdb挂载目录，/etc/fstab新增行：/dev/sdb1 /datas4hadoop ext4 defaults 0 2

运行用户hadoop:

        chown -R hadoop:hadoop /hadoop-2.2.0  /datas4hadoop

文件/etc/sudoers 新增行   hadoop  ALL=(ALL:ALL) ALL

hadoop01.ko:

时钟服务器：

         /etc/ntp.conf 新增行:  

server 127.127.1.0
fudge 127.127.1.0 stratum 1

ssh trust:

 hadoop01.ko免密码登录hadoop02.ko

 ssh-keygen -t rsa

        cat /home/hadoop/id_rsa.pub >> /home/hadoop/.ssh/authorized_keys (hadoop01.ko执行，hadoop01.ko ---->hadoop01.ko)

        cat /home/hadoop/id_rsa.pub >> /home/hadoop/.ssh/authorized_keys (hadoop02.ko执行，hadoop01.ko ---->hadoop02.ko)

        /home/hadoop/.ssh/known_hosts (hadoop01.ko ------->ssh hadoop01.ko, ssh hadoop02.ko     hadoop02.ko --------->ssh hadoop02.ko)   输入yes确认

/etc/hosts:

127.0.0.1       localhost
192.168.179.242 hadoop01.ko hadoop01
192.168.179.243 hadoop02.ko hadoop02

1、core-site.xml

<configuration>
         <property>
                <name>fs.defaultFS</name>
                <value>hdfs://hadoop01.ko:12000/</value>
        </property>
         <property>
                <name>io.file.buffer.size</name>
                <value>131072</value>
        </property>
                <property>
                <name>hadoop.tmp.dir</name>
                <value>/datas4hadoop</value>
        </property>
        <property>
                <name>mapreduce.randomwriter.bytespermap</name>
                <value>1048576</value>
        </property>
        <property>
                <name>mapreduce.randomwriter.mapsperhost</name>
                <value>1</value>
        </property>
</configuration>

2、hadoop-env.sh，yarn-env.sh

export JAVA_HOME=/hadoop-2.2.0/jdk1.8.0

3、hdfs-site.xml

<configuration>      

        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>hadoop01.ko:12010</value>

               <description>default: 0.0.0.0:50090</description>

        </property>

</configuration>

4、yarn-site.xml

<configuration>  

        <property>
                <name>yarn.acl.enable</name>
                <value>false</value>

               <description>default: true</description>
        </property>

        <property>
                <name>yarn.resourcemanager.address</name>
                <value>hadoop01.ko:12001</value>

               <description>yarn.resourcemanager.hostname default 0.0.0.0 default: ${yarn.resourcemanager.hostname}:8032</description>
        </property>
        <property>
                <name>yarn.resourcemanager.scheduler.address</name>
                <value>hadoop01.ko:12002</value>

               <description>default: ${yarn.resourcemanager.hostname}:8030</description>
        </property>
        <property>
                <name>yarn.resourcemanager.resource-tracker.address</name>
                <value>hadoop01.ko:12003</value>

               <description>default: ${yarn.resourcemanager.hostname}:8031</description>
        </property>
        <property>
                <name>yarn.resourcemanager.admin.address</name>
                <value>hadoop01.ko:12004</value>

               <description>default: ${yarn.resourcemanager.hostname}:8033</description>
        </property>
     <property>
                <name>yarn.resourcemanager.scheduler.class</name>
                <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
                <description>CapacityScheduler,FifoScheduler</description>
        </property>

       <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
                <value>org.apache.hadoop.mapred.ShuffleHandler</value>
        </property>

</configuration>

5、mapred-site.xml

<configuration>

        <property>
                <name>mapreduce.jobtracker.address</name>
                <value>hadoop01.ko:12020</value>

<description>default: local</description>
        </property>
        <property>
                <name>mapreduce.tasktracker.report.address</name>
                <value>hadoop01.ko:12023</value>

<description>default: 127.0.0.1:0</description>
        </property>

</configuration>

6、slaves

hadoop01.ko
hadoop02.ko

测试：

wordcount:

       mkdir input
       echo Hello world Bye world >>input/file1
       echo hello hadoop bye hadoop >> input/file2
       /hadoop-2.2.0/bin/hadoop fs -mkdir /tmp/input
       /hadoop-2.2.0/bin/hadoop fs -put input/* /tmp/input
       /hadoop-2.2.0/bin/hadoop fs -ls /tmp/input

/hadoop-2.2.0/bin/hadoop jar \ 

/hadoop-2.2.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.2.0.jar wordcount /tmp/input /tmp/output
      结果：
/hadoop-2.2.0/bin/hadoop fs -cat /tmp/output/part-r-00000

randomwriter:

/hadoop-2.2.0/bin/hadoop jar \ 

/hadoop-2.2.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.2.0.jar randomwriter out