centos7(vm)下hadoop2.7.2伪分布式安装验证(x86)

1.vm安装(略)

2.软件下载

hadoop-2.7.2.tar

jdk-8u77-linux-x64.tar

3.软件安装

a.java安装

[root@bogon hadoop]# which java
/root/java/jdk1.8.0_77/bin/java
[root@bogon hadoop]#

[root@bogon hadoop]# vi /etc/profile

#JAVA
export JAVA_HOME=/root/java/jdk1.8.0_77
export JRE_HOME=$JAVA_HOME/jre
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

[root@bogon hadoop]#source /etc/profile

b.hadoop安装

1)ssh 免密匙登录

[root@bogon ~]# ssh-keygen -t dsa
Generating public/private dsa key pair.
Enter file in which to save the key (/root/.ssh/id_dsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_dsa.
Your public key has been saved in /root/.ssh/id_dsa.pub.
The key fingerprint is:
aa:c8:e5:c1:b2:0b:af:29:d6:56:9f:e5:b7:1f:dd:fa root@bogon
The key's randomart image is:
+--[ DSA 1024]----+
|                 |
|                 |
|                 |
|                 |
|        S        |
|   . . . .    . .|
|. o = o +    . ..|
|.* O o o . .  .. |
|=.B.o     ..o...E|
+-----------------+
[root@bogon ~]# cd ~/.ssh/
[root@bogon .ssh]# ll
total 12
-rw-------. 1 root root 668 Dec 12 23:59 id_dsa
-rw-r--r--. 1 root root 600 Dec 12 23:59 id_dsa.pub
-rw-r--r--. 1 root root 171 Dec 12 23:58 known_hosts
[root@bogon .ssh]# cat known_hosts
localhost ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKHB/rPvh5KK1kPRB95v6OoCKUAp0f2f40baAkgMJ/Tmc/PSYAcoHhjTpgg/8P+YJ5/MCm7Qvvihqg9icMAn5ig=
[root@bogon .ssh]# cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
[root@bogon .ssh]# chmod 0600 ~/.ssh/authorized_keys
[root@bogon .ssh]# ssh localhost
Last login: Mon Dec 12 23:58:35 2016 from localhost

2)hadoop安装

[root@bogon hadoop]# tar -zxvf hadoop-2.7.2.tar.gz

[root@bogon hadoop-2.7.2]# pwd
/root/hadoop/hadoop-2.7.2
[root@bogon hadoop-2.7.2]# vi /etc/profile

#HADOOP
export HADOOP_HOME=/root/hadoop/hadoop-2.7.2
export PATH=.:$HADOOP_HOME/bin:$JAVA_HOME/bin:$PATH

wq!

[root@bogon hadoop-2.7.2]#source /etc/profile

[root@bogon hadoop-2.7.2]# bin/hadoop
Usage: hadoop [--config confdir] [COMMAND | CLASSNAME]
  CLASSNAME            run the class named CLASSNAME
 or
  where COMMAND is one of:

.........

3)修改hadoop-env.sh 增加JAVA_HOME路径

[root@bogon hadoop]# pwd
/root/hadoop/hadoop-2.7.2/etc/hadoop
[root@bogon hadoop]# vi hadoop-env.sh

# The java implementation to use.
export JAVA_HOME=/root/java/jdk1.8.0_77

wq!

4)编辑core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>

5)编辑hdfs-site.xml

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

6)格式化

[root@bogon hadoop-2.7.2]# bin/hdfs namenode -format
16/12/13 00:22:58 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG:   host = bogon/220.250.64.225
STARTUP_MSG:   args = [-format]
STARTUP_MSG:   version = 2.7.2

......

16/12/13 00:23:02 INFO common.Storage: Storage directory /tmp/hadoop-root/dfs/name has been successfully formatted.
16/12/13 00:23:02 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0
16/12/13 00:23:02 INFO util.ExitUtil: Exiting with status 0
16/12/13 00:23:02 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at bogon/220.250.64.225
************************************************************/

7)启动hdfs

[root@bogon hadoop-2.7.2]# sbin/start-dfs.sh
Starting namenodes on [localhost]
localhost: starting namenode, logging to /root/hadoop/hadoop-2.7.2/logs/hadoop-root-namenode-bogon.out
localhost: starting datanode, logging to /root/hadoop/hadoop-2.7.2/logs/hadoop-root-datanode-bogon.out
Starting secondary namenodes [0.0.0.0]
The authenticity of host '0.0.0.0 (0.0.0.0)' can't be established.
ECDSA key fingerprint is 98:b1:e5:3b:fd:ff:46:d2:10:ab:99:ab:f1:dd:ff:e8.
Are you sure you want to continue connecting (yes/no)? yes
0.0.0.0: Warning: Permanently added '0.0.0.0' (ECDSA) to the list of known hosts.
0.0.0.0: starting secondarynamenode, logging to /root/hadoop/hadoop-2.7.2/logs/hadoop-root-secondarynamenode-bogon.out
[root@bogon hadoop-2.7.2]#
[root@bogon hadoop-2.7.2]#
[root@bogon hadoop-2.7.2]# bin/hdfs dfs -mkdir /user
[root@bogon hadoop-2.7.2]# bin/hdfs dfs -mkdir /user/qiliansheng
[root@bogon hadoop-2.7.2]# bin/hadoop dfs -mkdir -p input
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.

[root@bogon hadoop-2.7.2]# ll
total 40
drwxr-xr-x. 2 10011 10011  4096 Jan 26  2016 bin
drwxr-xr-x. 3 10011 10011    19 Jan 26  2016 etc
drwxr-xr-x. 2 10011 10011   101 Jan 26  2016 include
drwxr-xr-x. 3 10011 10011    19 Jan 26  2016 lib
drwxr-xr-x. 2 10011 10011  4096 Jan 26  2016 libexec
-rw-r--r--. 1 10011 10011 15429 Jan 26  2016 LICENSE.txt
drwxr-xr-x. 2 root  root   4096 Dec 13 00:26 logs
-rw-r--r--. 1 10011 10011   101 Jan 26  2016 NOTICE.txt
-rw-r--r--. 1 10011 10011  1366 Jan 26  2016 README.txt
drwxr-xr-x. 2 10011 10011  4096 Jan 26  2016 sbin
drwxr-xr-x. 4 10011 10011    29 Jan 26  2016 share
[root@bogon hadoop-2.7.2]# bin/hdfs dfs -put etc/hadoop input
[root@bogon hadoop-2.7.2]#
[root@bogon hadoop-2.7.2]#
[root@bogon hadoop-2.7.2]# bin/hdfs dfs -ls
Found 1 items
drwxr-xr-x   - root supergroup          0 2016-12-13 00:35 input

8)配置mapred-site.xml

[root@bogon hadoop]# mv mapred-site.xml.template mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>

wq!

9)配置yarn-site.xml

[root@bogon hadoop]# vi mapred-site.xml 

   <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>

10)启动

[root@bogon hadoop-2.7.2]# sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [localhost]
localhost: starting namenode, logging to /root/hadoop/hadoop-2.7.2/logs/hadoop-root-namenode-bogon.out
localhost: starting datanode, logging to /root/hadoop/hadoop-2.7.2/logs/hadoop-root-datanode-bogon.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /root/hadoop/hadoop-2.7.2/logs/hadoop-root-secondarynamenode-bogon.out
starting yarn daemons
starting resourcemanager, logging to /root/hadoop/hadoop-2.7.2/logs/yarn-root-resourcemanager-bogon.out
localhost: starting nodemanager, logging to /root/hadoop/hadoop-2.7.2/logs/yarn-root-nodemanager-bogon.out

11)验证

[root@bogon hadoop-2.7.2]# jps
16307 SecondaryNameNode
16596 ResourceManager
15909 NameNode
16790 Jps
16702 NodeManager

12)