Hadoop 2.6.0 安装笔记

1.第一部分 Linux 安装

---

安装操作系统

挂载光盘

mkdir /mnt/rhel

cat << EOF > /etc/fstab

/dev/cdrom              /mnt/rhel               auto defaults       0 0

EOF

---

配置yum

cat << EOF > /etc/yum.repos.d/dvd.repo

[dvd]

name=dvd

baseurl=file:///mnt/rhel/Server

enabled=1

gpgcheck=0

EOF 

---

配置主机名

vim /etc/sysconfig/network

---

配置防火墙

service iptables stop

chkconfig iptables off 

---

卸载预装的java

[root@Node1 ~]# rpm -qa *java*

java-1.6.0-openjdk-1.6.0.0-1.50.1.11.5.el6_3.x86_64

tzdata-java-2012j-1.el6.noarch

java-1.7.0-openjdk-1.7.0.9-2.3.4.1.el6_3.x86_64

[root@Node1 ~]# rpm -e java-1.6.0-openjdk-1.6.0.0-1.50.1.11.5.el6_3.x86_64

[root@Node1 ~]# rpm -e java-1.7.0-openjdk-1.7.0.9-2.3.4.1.el6_3.x86_64

---

安装必要的软件

yum -y install ssh rsync

---

创建用户

useradd hadoop

echo hadoop|passwd --stdin hadoop

---

创建目录

mkdir -p /hadoop/software

chown -R hadoop:hadoop /hadoop

---

上传软件

java/jdk1.7.0_75

hadoop 2.6.0

---

解压安装java

tar -zvxf jdk-7u75-linux-x64.tar.gz

mv jdk1.7.0_75/ ..

cat << EOF >> /home/hadoop/.bash_profile

export JAVA_HOME=/hadoop/jdk1.7.0_75

export JAVA_BIN=/hadoop/jdk1.7.0_75/bin

export PATH=\$PATH:\$JAVA_HOME/bin

export CLASSPATH=.:\$JAVA_HOME/lib/dt.jar:\$JAVA_HOME/lib/tools.jar

EOF

---

安装hadoop

编译hadoop(略过，如果安装的是64位系统，需要编译替换lib库中的文件)

安装

---

配置

hadoop-env.sh

--添加

export JAVA_HOME=/hadoop/jdk1.7.0_75

export JAVA_BIN=/hadoop/jdk1.7.0_75/bin

export PATH=$PATH:$JAVA_HOME/bin

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

---

配置 core-site.xml

<configuration>

<property>

<name>fs.defaultFS</name>

<value>hdfs://hnamenode:9000</value>

</property>

<property>

<name>io.file.buffer.size</name>

<value>131072</value>

</property>

</configuration>

---

配置 hdfs-site.xml

<property>

  <name>dfs.namenode.name.dir</name>

  <value>/hadoop/hadoop-2.6.0/hdfs/name</value>

</property>

<property>

  <name>dfs.namenode.hosts</name>

  <value>hdatanode1,hdatanode2,hdatanode3</value>

</property>

<property>

  <name>dfs.blocksize</name>

  <value>67108864</value>

</property>

<property>

  <name>dfs.namenode.handler.count</name>

  <value>100</value>

</property>

<!-- DataNode related configuration -->

 

<property>

 <name>dfs.datanode.data.dir</name>

 <value>/hadoop/hadoop-2. 6.0/hdfs/data</value>

 <final>true</final>

</property>

---

配置 yarn-site.xml

<!-- ResourceManager and NodeManager related configuration ***BEGIN*** -->

<property>

 <name>yarn.acl.enable</name>

 <value>false</value>

</property>

<property>

 <name>yarn.admin.acl</name>

 <value>*</value>

</property>

<property>

 <name>yarn.log-aggregation-enable</name>

 <value>false</value>

</property>

<!-- ResourceManager and NodeManager related configuration ***END*** -->

<!-- ResourceManager related configuration ***BEGIN*** -->

<property>

<name>yarn.resourcemanager.address</name>

<value>hnamenode:18040</value>

</property>

<property>

<name>yarn.resourcemanager.scheduler.address</name>

<value>hnamenode:18030</value>

</property>

<property>

<name>yarn.resourcemanager.resource-tracker.address</name>

<value>hnamenode:18025</value>

</property>

<property>

<name>yarn.resourcemanager.admin.address</name>

<value>hnamenode:18141</value>

</property>

<property>

<name>yarn.resourcemanager.webapp.address</name>

<value>hnamenode:18088</value>

</property>

<property>

<name>yarn.resourcemanager.hostname</name>

<value>hnamenode</value>

</property>

<property>

<name>yarn.resourcemanager.scheduler.class</name>

<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>

</property>

<property>

    <name>yarn.scheduler.minimum-allocation-mb</name>

    <value>256</value>

</property>

<property>

    <name>yarn.scheduler.maximum-allocation-mb</name>                    //这个要大点，要超过 mapreduce.map.memory.mb

    <value>2048</value>

</property>

<!--

<property>

    <name>yarn.resourcemanager.nodes.include-path</name>

    <value></value>

</property>

<property>

    <name>yarn.resourcemanager.nodes.exclude-path</name>

    <value></value>

</property>

-->

<!-- ResourceManager related configuration ***END*** -->

<!-- NodeManager related configuration ***BEGIN*** -->

<property>

    <name>yarn.nodemanager.resource.memory-mb</name>           //这个也要大点

    <value>1024</value>

</property>

<property>

    <name>yarn.nodemanager.vmem-pmem-ratio</name>

    <value>2.1</value>

</property>

<property>

    <name>yarn.nodemanager.local-dirs</name>

    <value>/hadoop/hadoop-2. 6.0/yarn/local</value>

</property>    

<property>

    <name>yarn.nodemanager.log-dirs</name>

    <value>/hadoop/hadoop-2. 6.0/yarn/log</value>

</property>

<property>

    <name>yarn.nodemanager.log.retain-seconds</name>

    <value>10800</value>

</property>

<property>

    <name>yarn.nodemanager.remote-app-log-dir</name>

    <value>/hadoop/hadoop-2. 6.0/yarn/log-aggregation</value>

</property>

<property>

    <name>yarn.nodemanager.remote-app-log-dir-suffix</name>

    <value>logs</value>

</property>

<property>

<name>yarn.nodemanager.aux-services</name>

<value>mapreduce_shuffle</value>

</property>

<!-- NodeManager related configuration ***END*** -->

<!-- History Server related configuration ***BEGIN*** -->

<property>

    <name>yarn.log-aggregation.retain-seconds</name>

    <value>-1</value>

</property>

<property>

    <name>yarn.log-aggregation.retain-check-interval-seconds</name>

    <value>-1</value>

</property>

<!-- History Server related configuration ***END*** -->

---

配置

cp /hadoop/hadoop-2. 6.0/etc/hadoop/mapred-site.xml.template /hadoop/hadoop-2.6.0/etc/hadoop/mapred-site.xml

vim /hadoop/hadoop-2. 6.0/etc/hadoop/mapred-site.xml

<!-- MapReduce Applications related configuration ***BEGIN*** -->

<property>

<name>mapreduce.framework.name</name>

<value>yarn</value>

</property>

<property>

<name>mapreduce.map.memory.mb</name>               //这个也要大点 用默认 1536吧

<value>1536</value>

</property>

<property>

<name>mapreduce.map.java.opts</name>

<value>-Xmx1024M</value>

</property>

<property>

<name>mapreduce.reduce.memory.mb</name>            //这个也要大点 用默认 3072

<value>2048</value>

</property>

<property>

<name>mapreduce.reduce.java.opts</name>

<value>-Xmx2048M</value>

</property>

<property>

<name>mapreduce.task.io.sort.mb</name>

<value>512</value>

</property>

<property>

<name>mapreduce.task.io.sort.factor</name>

<value>10</value>

</property>

<property>

<name>mapreduce.reduce.shuffle.parallelcopies</name>

<value>20</value>

</property>

<!-- MapReduce Applications related configuration ***END*** -->

<!-- MapReduce JobHistory Server related configuration ***BEGIN*** -->

<property>

    <name>mapreduce.jobhistory.address</name>

    <value>hnamenode:10020</value>

</property>

<property>

    <name>mapreduce.jobhistory.webapp.address</name>

    <value>hnamenode:19888</value>

</property>

<property>

    <name>mapreduce.jobhistory.intermediate-done-dir</name>

    <value>/hadoop/hadoop-2. 6.0/mr_history/tmp</value>

</property>

<property>

    <name>mapreduce.jobhistory.done-dir</name>

    <value>/hadoop/hadoop-2. 6.0/mr_history/done</value>

</property>

<!-- MapReduce JobHistory Server related configuration ***END*** -->

---

配置slaves

vim /hadoop/hadoop-2. 6.0/etc/hadoop/slaves

hdatanode1

hdatanode2

hdatanode3

---

配置时间同步

(略)

---

配置

/etc/hosts

cat << EOF > /etc/hosts

#127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4

#::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.28.4 Node1

192.168.28.6 hdatanode1

192.168.28.7  hdatanode2

192.168.28.8  hdatanode3

EOF

---

配置ssh(namenode 联通 datanode)

#每个节点都要做

su - hadoop

ssh-keygen -t rsa 

ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@Node1

ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@hdatanode1

ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@hdatanode2

ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@hdatanode3

---

########### 验证namenode datanode ########### 

/hadoop/hadoop-2.6.0/bin/hadoop namenode

/hadoop/hadoop -2.6.0/bin/hadoop datanode

########### 验证resourcemanager  nodemanager ###########

yarn resourcemanager

yarn nodemanager

hdfs namenode -format pyc

$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode

$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start datanode

$HADOOP_PREFIX/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager

$HADOOP_PREFIX/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager

$HADOOP_PREFIX/sbin/yarn-daemon.sh start proxyserver --config $HADOOP_CONF_DIR

$HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver --config $HADOOP_CONF_DIR

$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop namenode

$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop datanode

$HADOOP_PREFIX/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager

$HADOOP_PREFIX/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager

$HADOOP_PREFIX/sbin/yarn-daemon.sh stop proxyserver --config $HADOOP_CONF_DIR

$HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh stop historyserver --config $HADOOP_CONF_DIR

$HADOOP_PREFIX/sbin/start-all.sh

--启动hadoop

su - hadoop

source /hadoop/hadoop-2.5.2/etc/hadoop/hadoop-env.sh

$HADOOP_PREFIX/sbin/start-all.sh

su - hadoop

export DISPLAY=192.168.28.1:0.0

xhost +

cd /hadoop/eclipse

./eclipse

$HADOOP_PREFIX/sbin/stop-all.sh

启动方式

start-all.sh

start-dfs.sh

start-yarn.sh

--测试程序

bin/hadoop dfs -put /hadoop/hadoop-1.2.1/input hdfs://hnamenode:9000/user/hadoop/in

bin/hadoop dfs -ls ./in/*

bin/hadoop jar hadoop-examples-1.2.1.jar wordcount hdfs://hnamenode:9000/user/hadoop/in hdfs://hnamenode:9000/user/hadoop/out

hadoop jar hadoop-examples-1.2.1.jar wordcount in out