Hadoop——HA-HDFS搭建
哔哩哔哩视频详解:https://www.bilibili.com/video/av75781441
hadoop-env.sh
export JAVA_HOME=/root/software/jdk1.8.0_181
core-site.xml
<property>
<name>hadoop.tmp.dir</name>
<value>/root/software/hadoop_tmp</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://masters</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>master:2181,slave1:2181,slave2:2181</value>
</property>
hdfs-site.xml
<property>
<name>dfs.nameservices</name>
<value>masters</value>
</property>
<property>
<name>dfs.ha.namenodes.masters</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.masters.nn1</name>
<value>master:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.masters.nn1</name>
<value>master:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.masters.nn2</name>
<value>slave1:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.masters.nn2</name>
<value>slave1:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://master:8485;slave1:8485;slave2:8485/masters</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.masters</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/root/software/hadoop_tmp/journaldata</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
将配置文件发送给slaves
scp -r hadoop slave1:/root/software/
scp -r hadoop slave2:/root/software/
slaves
slave1
slave2
启动journalnode
hadoop-daemon.sh start journalnode
格式化namenode
hdfs namenode -format
将元数据发送给其他的namenode
scp -r dfs/ slave1:/root/software/hadoop_tmp/
格式化ZKFC
hdfs zkfc -formatZK
启动HDFS
start-dfs.sh
mapred-site.xml
<!-- 通知框架MR使用YARN -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
yarn-site.xml
<!-- NodeManager上运行的附属服务。需配置成mapreduce_shuffle,才可运行MapReduce程序 -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
将配置文件发送给slaves
scp -r hadoop/etc/hadoop/mapred-site.xml slave1:/root/software/hadoop/etc/hadoop
scp -r hadoop/etc/hadoop/yarn-site.xml slave1:/root/software/hadoop/etc/hadoop
scp -r hadoop/etc/hadoop/mapred-site.xml slave2:/root/software/hadoop/etc/hadoop
scp -r hadoop/etc/hadoop/yarn-site.xml slave2:/root/software/hadoop/etc/hadoop
启动Yarn
start-yarn.sh