CentOS6.4安装Hadoop2.1beta

1.安装JDK7 rpm到/usr/java/jdk1.7.0_40
[code="java"]# rpm -ivh jdk-7u40-linux-x64.rpm
Preparing... ########################################### [100%]
1:jdk ########################################### [100%]
Unpacking JAR files...
rt.jar...
jsse.jar...
charsets.jar...
tools.jar...
localedata.jar...
jfxrt.jar...
# java -version
java version "1.7.0_40"
Java(TM) SE Runtime Environment (build 1.7.0_40-b43)
Java HotSpot(TM) 64-Bit Server VM (build 24.0-b56, mixed mode)[/code]

2.下载hadoop2.1beta并解压至/opt/hadoop-2.1.0-beta，并建立软链/opt/hadoop接到/opt/hadoop-2.1.0-beta
[code="java"]# cd /opt
# tar -zxvf /root/hadoop-2.1.0-beta.tar.gz
# ln -s hadoop-2.1.0-beta hadoop
# ls
cedar hadoop hadoop-2.1.0-beta rh[/code]
3. 设置环境变量、
[code="java"]# vim /etc/profile[/code]
在文件末尾添加如下内容:

export JAVA_HOME=/usr/java/default
export PATH=${JAVA_HOME}/bin:${PATH}

export HADOOP_PREFIX=/opt/hadoop
export PATH=$PATH:$HADOOP_PREFIX/bin
export PATH=$PATH:$HADOOP_PREFIX/sbin
export HADOOP_MAPRED_HOME=${HADOOP_PREFIX}
export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
export YARN_HOME=${HADOOP_PREFIX}

使上述修改生效:# source /etc/profile
4.建立用户和组
[code="java"]
# groupadd hadoop
# useradd -g hadoop yarn
# useradd -g hadoop hdfs
# useradd -g hadoop mapred[/code]
5.创建相关路径和文件
[code="java"]# mkdir -p /var/data/hadoop/hdfs/nn
# mkdir -p /var/data/hadoop/hdfs/snn
# mkdir -p /var/data/hadoop/hdfs/dn
# mkdir -p /opt/hadoop/logs
# mkdir -p /var/log/hadoop/yarn
# cd /opt/hadoop/logs
# chown hdfs:hadoop /var/data/hadoop/hdfs -R
# chown yarn:hadoop /var/log/hadoop/yarn -R
# chmod g+w /opt/hadoop/logs
# chown yarn:hadoop . -R[/code]
6.修改hadoop配置文件
vim /opt/hadoop/etc/hadoop/core-site.xml

<configuration>
  <property>
    <name>fs.default.name</name>
    <value>hdfs://192.168.32.31:8020</value>
  </property>
  <property>
    <name>hadoop.http.staticuser.user</name>
    <value>hdfs</value>
  </property>
</configuration>

7.修改hdfs-site.xml
vim /opt/hadoop/etc/hadoop/hdfs-site.xml

<configuration>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/var/data/hadoop/hdfs/nn</value>
  </property>
  <property>
    <name>fs.checkpoint.dir</name>
    <value>file:/var/data/hadoop/hdfs/snn</value>
  </property>
  <property>
    <name>fs.checkpoint.edits.dir</name>
    <value>file:/var/data/hadoop/hdfs/snn</value>
  </property>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>file:/var/data/hadoop/hdfs/dn</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <property>
    <name>dfs.permissions</name>
    <value>false</value>
  </property>
</configuration>

8.修改mapred-site.xml
#cp mapred-site.xml.template mapred-site.xml
#vi mapred-site.xml

<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
</configuration>

9.修改yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce.shuffle</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
  </property>
</configuration>

格式化集群

su - hdfs
cd /opt/hadoop
./hdfs namenode -format

启动hdfs

cd ../sbin
./hadoop-daemon.sh start namenode
./hadoop-daemon.sh start secondarynamenode
./hadoop-daemon.sh start datanode

启动yarn

su - yarn
cd /opt/hadoop/sbin
./yarn-daemon.sh start resourcemanager
./yarn-daemon.sh start nodemanager