hadoop全分布式搭建

Hadoop全分布式

前提准备好三台服务器

关闭防火墙

[root@master ~]# systemctl stop firewalld
[root@master ~]# systemctl disable firewalld

添加主机映射

[root@master ~]# vim /etc/hosts

分发映射

[root@master ~]# scp /etc/hosts slave1:/etc/hosts
[root@master ~]# scp /etc/hosts slave2:/etc/hosts

配置免密登录

[root@master ~]# ssh-keygen
[root@slave1 ~]# ssh-keygen
[root@slave2 ~]# ssh-keygen

拷贝公钥

[root@master ~]# ssh-copy-id slave1
[root@master ~]# ssh-copy-id slave2
[root@master ~]# ssh-copy-id master
[root@slave1 ~]# ssh-copy-id master
[root@slave1 ~]# ssh-copy-id slave1
[root@slave1 ~]# ssh-copy-id slave2
[root@slave2 ~]# ssh-copy-id master
[root@slave2 ~]# ssh-copy-id slave1
[root@slave2 ~]# ssh-copy-id slave2

测试免密登录是否配置成功（登录成功后记得退出）

[root@master ~]# ssh slave1

解压安装包

[root@master software]# tar -zxvf jdk-8u144-linux-x64.tar.gz -C /usr/local/src/
[root@master software]# tar -zxvf hadoop-2.6.0.tar.gz -C /usr/local/src/

安装包改名

[root@master src]# mv hadoop-2.6.0/ hadoop
[root@master src]# mv jdk1.8.0_144/ java

配置环境变量

[root@master src]# vim /etc/profile
###java
export JAVA_HOME=/usr/local/src/java
export PATH=$PATH:$JAVA_HOME/bin
###hadoop
export HADOOP_HOME=/usr/local/src/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin

分发环境变量

[root@master src]# scp /etc/profile slave1:/etc/profile
[root@master src]# scp /etc/profile slave2:/etc/profile

使环境变量生效

[root@master src]# souce /etc/profile
[root@slave1 ~]# source /etc/profile
[root@slave2 ~]# source /etc/profile

修改配置文件

进入目录

[root@master ~]# cd /usr/local/src/hadoop/etc/hadoop/

core-site.xml

[root@master hadoop]# vim core-site.xml 
<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://master:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/usr/local/src/hadoop/tmp</value>
        </property>
</configuration>

hdfs-site.xml

[root@master hadoop]# vim hdfs-site.xml 
<configuration>
        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>master:50090</value>
        </property>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>/usr/local/src/hadoop/name</value>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/usr/local/src/hadoop/data</value>
        </property>
</configuration>

yarn-site.xml

[root@master hadoop]# vim yarn-site.xml 
<configuration>
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
        <property>
                <name>yarn.resourcemanager.hostname</name>
                <value>master</value>
        </property>
        <property>
                <name>yarn.log-aggregation-enable</name>
                <value>true</value>
        </property>
        <property>
                <name>yarn.log-aggregation.retain-seconds</name>
                <value>604800</value>
        </property>
</configuration>

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-5OULbPti-1637286518540)(https://cdn.nlark.com/yuque/0/2021/png/12508699/1625539140615-965896eb-a958-443d-8e71-01aefad47076.png)]

mapred-site.xml

[root@master hadoop]# mv mapred-site.xml.template mapred-site.xml
[root@master hadoop]# vim mapred-site.xml 
<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.address</name>
                <value>master:10020</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.webapp.address</name>
                <value>master:19888</value>
        </property>
</configuration>

修改JAVA_HOME

[root@master hadoop]# vim hadoop-env.sh
[root@master hadoop]# vim yarn-env.sh
[root@master hadoop]# vim mapred-env.sh 

声明集群

[root@master hadoop]# vim slaves 

分发集群

将配置好的hadoop和java分发到其他两台机器上

[root@master src]# scp -r /usr/local/src/hadoop slave1:/usr/local/src
[root@master src]# scp -r /usr/local/src/hadoop slave2:/usr/local/src
[root@master src]# scp -r /usr/local/src/java slave1:/usr/local/src
[root@master src]# scp -r /usr/local/src/java slave2:/usr/local/src

集群启动

格式化namenode

[root@master ~]# hdfs namenode -format

格式化成功

群起集群

[root@master ~]# start-all.sh 

[root@master ~]# jps

[root@slave1 ~]# jps

[root@slave2 ~]# jps

集群启动成功