Hadoop 由伪分布式到完全分布式搭建

     整体规划 namenode在node01节点，secondarynamenode 在node02节点，node02、node03、node04节点作为datanode。

1.修改配置文件$HADOOP_HOME/etc/hadoop/core-site.xml，将临时文件、namenode、datanode等的存储的目录做了调整，目的是和伪分布式做区分。

<configuration>
  <!-- 指定访问HDFS的时候路径的默认前缀  如果输入/  代表hdfs://node01:9000/ -->
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://node01:9000</value>
  </property>
  <!-- 指定指定临时文件存储目录 -->
  <property>
    <name>hadoop.tmp.dir</name>
    <value>/var/bdp/hadoop/full</value>
  </property>
  <!-- 指定namenode存储目录 -->
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>/var/bdp/hadoop/full/dfs/name</value>
  </property>
  <!-- 指定datanode存储目录 -->
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>/var/bdp/hadoop/full/dfs/data</value>
  </property>
  <!-- 指定secondarynamenode存储目录 -->
  <property>
    <name>dfs.namenode.checkpoint.dir</name>
    <value>/var/bdp/hadoop/full/dfs/secondary</value>
  </property>
</configuration>

2.修改$HADOOP_HOME/etc/hadoop/hdfs-site.xml 配置文件中副本数和SNN节点

<configuration>
  <!-- 指定block副本数 -->
  <property>
    <name>dfs.replication</name>
    <value>2</value>
  </property>
  <!-- 指定secondarynamenode所在的位置 -->
  <property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>node02:50090</value>
  </property>
</configuration>

3.修改配置文件$HADOOP_HOME/etc/hadoop/slaves更改datanode节点

[root@node01 hadoop]# vi slaves 
[root@node01 hadoop]# cat slaves 
node02
node03
node04

4.将hadoop 文件分发给node02、node03、node04并格式化namenode

[root@node01 bdp]# pwd
/opt/bdp
[root@node01 bdp]# scp -r hadoop-2.6.5 node02:`pwd`
[root@node01 bdp]# scp -r hadoop-2.6.5 node03:`pwd`
[root@node01 bdp]# scp -r hadoop-2.6.5 node04:`pwd`
[root@node01 bdp]# hdfs namenode -format

5.启动

[root@node01 bdp]# start-dfs.sh 
Starting namenodes on [node01]
node01: starting namenode, logging to /opt/bdp/hadoop-2.6.5/logs/hadoop-root-namenode-node01.out
node03: starting datanode, logging to /opt/bdp/hadoop-2.6.5/logs/hadoop-root-datanode-node03.out
node04: starting datanode, logging to /opt/bdp/hadoop-2.6.5/logs/hadoop-root-datanode-node04.out
node02: starting datanode, logging to /opt/bdp/hadoop-2.6.5/logs/hadoop-root-datanode-node02.out
Starting secondary namenodes [node02]
node02: starting secondarynamenode, logging to /opt/bdp/hadoop-2.6.5/logs/hadoop-root-secondarynamenode-node02.out

[root@node01 bdp]# jps
8165 NameNode
8760 Jps
[root@node02 bdp]# jps
8210 Jps
7701 DataNode
7773 SecondaryNameNode
[root@node03 bdp]# jps
7632 DataNode
8077 Jps
[root@node04 ~]# jps
7677 DataNode
8126 Jps


node01查看集群idclusterID
[root@node01 bdp]# cat /var/bdp/hadoop/full/dfs/name/current/VERSION 
#Thu Jul 07 09:24:06 CST 2022
namespaceID=1307364786
clusterID=CID-778eadaa-ee65-46d3-83f4-1f5c67e026a3
cTime=0
storageType=NAME_NODE
blockpoolID=BP-1240158689-192.168.23.101-1657157046253
layoutVersion=-60

node02~node04查看集群idclusterID
[root@node02 bdp]# cat /var/bdp/hadoop/full/dfs/name/current/VERSION 
cat: /var/bdp/hadoop/full/dfs/name/current/VERSION: No such file or directory
[root@node02 bdp]# cat /var/bdp/hadoop/full/dfs/data/current/VERSION 
#Thu Jul 07 09:27:56 CST 2022
storageID=DS-1c4aae55-a55f-46ad-ae2e-e0e002bec08b
clusterID=CID-778eadaa-ee65-46d3-83f4-1f5c67e026a3
cTime=0
datanodeUuid=0c630270-38a1-43d8-aec2-a9b1a8ca0ad3
storageType=DATA_NODE
layoutVersion=-56

确认4台服务器 集群ID相同