本篇文章搭建搭建在hadoop集群和zookeeper集群基础之上,若hadoop集群没有搭建请参照:hadoop集群搭建,若zookeeper集群没有搭建,请参照:zookeeper集群搭建
一、修改配置文件
以下操作在主节点进行,首先进入hadoop目录:cd /opt/hadoop260
- 修改core-site.xml
vi core-site.xml
<configuration>
<!-- 指定hdfs的nameservice为ns -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://ns</value>
</property>
<!-- 指定hadoop临时目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/hadoop260/tmp</value>
</property>
<!-- 设置所有用户都当做root用户,所有组都当做root组 -->
<property>
<name>hadoop.proxyuser.root.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
</property>
<!-- 指定zookeeper地址 -->
<property>
<name>ha.zookeeper.quorum</name>
<value>hadoop1:2181,hadoop2:2181,hadoop3:2181</value>
</property>
</configuration>
- hdfs-site.xml
<configuration>
<!--节点数-->
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<!--指定名称服务,与core-site.xml中的一致-->
<property>
<name>dfs.nameservices</name>
<value>ns</value>
</property>
<!--指定名称服务下的namenode节点-->
<property>
<name>dfs.ha.namenodes.ns</name>
<value>hadoop1,hadoop2</value>
</property>
<!-- hadoop1的RPC通信地址 -->
<property>
<name>dfs.namenode.rpc-address.ns.hadoop1</name>
<value>hadoop1:9000</value>
</property>
<!-- hadoop2的RPC通信地址 -->
<property>
<name>dfs.namenode.rpc-address.ns.hadoop2</name>
<value>hadoop2:9000</value>
</property>
<!-- hadoop1的http通信地址 -->
<property>
<name>dfs.namenode.http-address.ns.hadoop1</name>
<value>hadoop1:50070</value>
</property>
<!-- hadoop2的http通信地址 -->
<property>
<name>dfs.namenode.http-address.ns.hadoop2</name>
<value>hadoop2:50070</value>
</property>
<!-- 指定NameNode的元数据在JournalNode上的存放位置 -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hadoop1:8485;hadoop2:8485;hadoop3:8485/ns</value>
</property>
<!-- 开启NameNode失败自动切换 -->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!-- 配置失败自动切换实现方式 -->
<property>
<name>dfs.client.failover.proxy.provider.ns</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔离机制方法,多个机制用换行分割,即每个机制暂用一行-->
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>
<!--当使用HA配置集群,则不需要secondarynamenode,下面内容需要手动注释
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>192.168.233.134:50090</value>
</property>
-->
- 修改yarn-site.xml
<!--HA配置-->
<!--开启-->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<!--指定rm集群id-->
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>cluster-id</value>
</property>
<!--指定rm集群的名称服务-->
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<!--指定rm映射关系-->
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>hadoop1</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>hadoop2</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>hadoop1:8088</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>hadoop2:8088</value>
</property>
<!--配置zookeeper集群-->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>hadoop1:2181,hadoop2:2181,hadoop3:2181</value>
</property>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<!-- 日志聚集功能使用 -->
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<!-- 日志保留时间设置7天 -->
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>604800</value>
</property>
<!-- 非HA模式下指定resourcemanager,需要手动注释
<property>
<name>yarn.resourcemanager.hostname</name>
<value>hadoop1</value>
</property>
-->
- 将主节点修改的内容传给子节点
scp /opt/hadoop260/etc/hadoop/*.xml hadoop2@root:/opt/hadoop260/etc/hadoop/
scp /opt/hadoop260/etc/hadoop/*.xml hadoop3@root:/opt/hadoop260/etc/hadoop/
二、启动集群
- 确保集群处于关闭状态,可用jps参看进程,若没有关闭,在主节点上执行
stop-all.sh
- 删除各个节点/opt/hadoop260目录下的logs和tmp文件
rm -rf logs
rm -rf tmp
- 三个节点手动启动journalnode
hadoop-daemon.sh start journalnode
- 在主节点上格式化hdfs
hadoop namenode -format
- 将主节点/opt/hadoop260/下的tmp传给备份节点hadoop2
scp -r /opt/hadoop260/tmp hadoop2@root:/opt/hadoop260/
- 三个节点启动zookeeper
zkServer.sh start
- 主节点开启hadoop
start-all.sh
- 主节点格式化zkfc
hdfs zkfc -formatZK
- 三个节点都开启zkfc
hadoop-daemon.sh start zkfc
- 备份节点hadoop2启动resourcemanager
yarn-daemon.sh start resourcemanager
三、查看集群部署
- hadoop1:
- hadoop2:
- hadoop3:
- 查看集群状态