Hadoop伪分布式环境搭建(自用流程

修改Centos7静态IP

vi /etc/sysconfig/network-scripts/ifcfg-ens33

BOOTPROTO=static
ONBOOT=yes
GATEWAY=192.168.17.1	//子网掩码
IPADDR=192.168.17.153
NETMASK=255.255.255.0
DNS1=218.85.157.99
DNS2=218.85.152.99
ZONE=public

systemctl restart network //重启网络

安装JDK

yum list installed | grep java          //查看jdk版本
yum -y remove java-版本-openjdk*        //卸载所有openjdk相关文件输入
yum -y remove tzdata-java.noarch        //卸载tzdata-java

---

yum search java |grep -i --color jdk        //查看JDK软件包
yum install -y java-版本-openjdk*           //选择jdk版本安装
java -version                              //jdk是否安装成功

---

vim /etc/profile //进行添加命令，jdk默认安装路径为/usr/lib/jvm

JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.302.b08-0.el7_9.x86_64
PATH=$PATH:$JAVA_HOME/bin         
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar         
export JAVA_HOME  CLASSPATH  PATH
//     jvm/后填写为当前安装的Java版本包，并非jer版本包

---

source /etc/profile                      //刷新profile文件使命令生效
java -version                            //复查jdk版本

---

进入Hadoop官网下载hadoop（此次使用并非源代码安装）

利用Xshell或在虚拟机中进行文件传输（相关命令，rz)

tar -zxvf hadoop-版本号.tar.gz -C /usr/hadoop //解压安装包到usr/hadoop下

Hadoop配置

systemctl stop firewalld		  (临时关闭防火墙)
systemctl disable firewalld		  (永久关闭)

---

//路径为/usr/hadoop/hadoop-3.3.1/etc/hadoop
vim core-site.xml

<configuration>
    <property>
            <name>fs.default.name</name>
            <value>hdfs://**master**:9000</value>
    </property>
    <property>
            <name>io.file.buffer.size</name>
            <value>131072</value>
    </property>
    <property>
            <name>hadoop.tmp.dir</name>
            <value>/root/hadoopdata</value>
            <description>Abasefor other temporary directories.</description>
    </property>
</configuration>

此处master为主机映射后的IP，非必须为maser
vim mapred-site.xml

<configuration>
    <property>
            <name>mapreduce.framework.name</name>
            <value>yarn</value>
    </property>
    <property>
 	<name>mapreduce.application.classpath</name>
	<value>/usr/hadoop/hadoop-3.3.1/share/hadoop/mapreduce/*,/usr/hadoop/hadoop-3.3.1/share/hadoop/maperduce/lib/*</value> 	//前为Hadoop安装位置，可能与他人不同，注意分辨
    </property>
	</configuration>

---

vim hdfs-site.xml

<configuration>
    <property>
            <name>dfs.replication</name>
            <value>2</value>
    </property>
</configuration>

---

vim yarn-site.xml

<configuration>
<!-- Site specific YARN configuration properties -->
<property>
    <name>yarn.resourcemanager.hostname</name>
    <value>master</value>
</property>
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
</configuration>

---

vim workers

localhost
名字2	
名字3
//其余2台机器

---

免密登录：

ssh-keygen -b 1024 -t rsa	     //后需3次回车操作
ssh-copy -id 192.168.222.163	 //ip为我的master地址,其余2从台从机同样输入此ip

---

添加用户
//路径为/usr/hadoop/hadoop-3.3.1/sbin

vim start-dfs.sh

#!/usr/bin/env bash
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

vim stop-dfs.sh

#!/usr/bin/env bash
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

vim start-yarn.sh

#!/usr/bin/env bash
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

vim stop-yarn.sh

#!/usr/bin/env bash
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

格式化

hadoop namenode -format 

vim /usr/hadoop/hadoop-3.3.1/etc/hadoop/hadoop-env.sh (位于37行

JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.302.b08-0.el7_9.x86_64

---

复制到其余2台从机上
scp root@目标ip或映射后的主机名:/usr/hadoop/hadoop-3.3.1/ /usr/hadoop

//一主二从都需要添加环境变量
vim /etc/profile //添加环境变量

export HADOOP_HOME=/usr/hadoop/hadoop-3.3.1
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

---

source /etc/profile                    //刷新profile文件
hadoop version                         //检查安装是否成功
		若成功显示hadoop版本号，则环境搭配成功

---

start-all.sh		//在hadoop的bin目录下启动