Hadoop——HA-HDFS搭建

Hadoop——HA-HDFS搭建

哔哩哔哩视频详解：https://www.bilibili.com/video/av75781441

hadoop-env.sh

export JAVA_HOME=/root/software/jdk1.8.0_181

core-site.xml

<property>
<name>hadoop.tmp.dir</name>
<value>/root/software/hadoop_tmp</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://masters</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>master:2181,slave1:2181,slave2:2181</value>	
</property>

hdfs-site.xml

<property>
<name>dfs.nameservices</name>     
<value>masters</value>
</property>
<property>
<name>dfs.ha.namenodes.masters</name>  
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.masters.nn1</name>
<value>master:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.masters.nn1</name>
<value>master:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.masters.nn2</name>
<value>slave1:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.masters.nn2</name>
<value>slave1:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://master:8485;slave1:8485;slave2:8485/masters</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.masters</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/root/software/hadoop_tmp/journaldata</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>

将配置文件发送给slaves

scp -r hadoop slave1:/root/software/
scp -r hadoop slave2:/root/software/

slaves

slave1
slave2

启动journalnode

hadoop-daemon.sh start journalnode

格式化namenode

hdfs namenode -format

将元数据发送给其他的namenode

scp -r dfs/ slave1:/root/software/hadoop_tmp/

格式化ZKFC

hdfs zkfc -formatZK

启动HDFS

start-dfs.sh

mapred-site.xml

<!-- 通知框架MR使用YARN -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

yarn-site.xml

<!-- NodeManager上运行的附属服务。需配置成mapreduce_shuffle，才可运行MapReduce程序 -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

将配置文件发送给slaves

scp -r hadoop/etc/hadoop/mapred-site.xml slave1:/root/software/hadoop/etc/hadoop
scp -r hadoop/etc/hadoop/yarn-site.xml slave1:/root/software/hadoop/etc/hadoop
scp -r hadoop/etc/hadoop/mapred-site.xml slave2:/root/software/hadoop/etc/hadoop
scp -r hadoop/etc/hadoop/yarn-site.xml slave2:/root/software/hadoop/etc/hadoop

启动Yarn

start-yarn.sh