Hadoop完全分布式集群搭建

1. 配置hadoop集群所需要的准备：a.三台虚拟机，这里我们分别命名为master,slave1,slave2；b.jdk安装包，Hadoop集群需要在jdk环境下运行，3.hadoop安装包。

2. 开始搭建（这里我们自己从第二步开始，就是解压jdk，Hadoop将h3cu下的jdk,hadoop解压到/usr/local/src/目录下）

[root@master /]#tar -zxvf /h3cu/jdk-8u161-linux-x64.tar.gz -C /usr/local/src/

[root@master /]#tar -zxvf /h3cu/hadoop-3.1.3.tar.gz -C /usr/local/src/

3进入/usr/local/src/目录进行改名

[root@master src]# mv jdk1.8.0_161/ jdk
[root@master src]# mv hadoop-3.1.3/ hadoop

3. 配置环境变量(根据要求选择/etc/profile 或者/root/.bash_profile)

[root@master src]# vi /etc/profile

export JAVA_HOME=/usr/local/src/jdk
export PATH=$PATH:$JAVA_HOME/bin
export HADOOP_HOME=/usr/local/src/hadoop
export HDFS_NAMENODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export HDFS_DATANODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

[root@master src]# source /etc/profile

4. 生成密钥，分发密钥（每台虚拟机操作一次，执行3遍）

[root@master src]# ssh-keygen -t rsa   #连按3次回车

[root@master src]# ssh-copy-id master
[root@master src]# ssh-copy-id slave1
[root@master src]# ssh-copy-id slave2

5. 分发环境变量配置文件到slave1，slave2

[root@master src]# scp  /etc/profile slave1:/etc/
[root@master src]# scp  /etc/profile slave2:/etc/
在slave1，slave2上使用命令：source /etc/profile使环境生效

6.分发jdk到slave1，slave2

[root@master src]# scp -r /usr/local/src/jdk/ slave1:/usr/local/src/
[root@master src]# scp -r /usr/local/src/jdk/ slave2:/usr/local/src/
在其中两台虚拟机上使用java -version命令出现一下便证明jdk分发成功
[root@slave1 /]# java -version
java version "1.8.0_161"
Java(TM) SE Runtime Environment (build 1.8.0_161-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)

8.进入/hadoop/etc/hadoop

[root@master src]# cd /usr/local/src/hadoop/etc/hadoop/

9.开始编辑配置文件

[root@master hadoop]# vi hadoop-env.sh
# The java implementation to use. By default, this environment
# variable is REQUIRED on ALL platforms except OS X!
export JAVA_HOME=/usr/local/src/jdk

# Location of Hadoop.  By default, Hadoop will attempt to determine
# this location based upon its execution path.

[root@master hadoop]# vi core-site.xml 
<configuration>
<property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
</property>
<property>
        <name>hadoop.tmp.dir</name>
        <value>/usr/local/src/hadoop/tmp</value>
</property>
</configuration>

[root@master hadoop]# vi hdfs-site.xml
<configuration>
<property>
        <name>dfs.replication</name>
        <value>1</value>
</property>
</configuration>


[root@master hadoop]# vi mapred-site.xml 
<configuration>
<property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
</property>
</configuration>

[root@master hadoop]# vi yarn-site.xml
<configuration>
<property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
</property>
<property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
</property>
<property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>false</value>
</property>
<property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
</property>
</configuration>

[root@master hadoop]# vi workers
master
slave1
slave2

10.分发hadoop到slave1，slave2

[root@master hadoop]# scp -r /usr/local/src/hadoop/ slave1:/usr/local/src/
[root@master hadoop]# scp -r /usr/local/src/hadoop/ slave2:/usr/local/src/

11.分发完成进行格式化（在master上）

[root@master hadoop]# hdfs namenode -format

12.启动Hadoop集群

[root@master hadoop]# start-all.sh

13.最后查看节点

[root@master hadoop]# jps
1696 SecondaryNameNode
1413 DataNode
1238 NameNode
2550 Jps
2185 NodeManager
2015 ResourceManager

[root@slave1 /]# jps
609 NodeManager
725 Jps
493 DataNode

[root@slave2 /]# jps
418 DataNode
534 NodeManager
670 Jps