Hadoop高可用集群搭建

集群资源与角色规划

node1	node2	node3	node4	node5
	zookeeper	zookeeper	zookeeper
nn1	nn2
datanode	datanode	datanode	datanode	datanode
	journal	journal	journal
		rm1	rm2
nodemanager	nodemanager	nodemanager	nodemanager	nodemanager

一、安装zookeeper

• zookeeper安装目录：/home/hadoop/zookeeper-3.4.13/
• zookeeper dataDir: /home/hadoop/zookeeper-data/data
• zookeeper dataLogDir: /home/hadoop/zookeeper-data/dataLog
• zoo.cfg 配置文件参考如下：

# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
dataDir=/home/hadoop/zookeeper-data/data
dataLogDir=/home/hadoop/zookeeper-data/dataLog
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the 
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
server.1=node3:2888:3888
server.2=node4:2888:3888
server.3=node5:2888:3888

• 在node3,node4,node5节点中创建myid文件：

cd /home/hadoop/zookeeper-data/data
touch myid
echo 1 > myid ----> node3
(echo 2 > myid ) -----> node4
(echo 3 > myid ) -----> node5

• 编辑zookeeper环境变量

export ZOOKEEPER_HOME=/home/hadoop/zookeeper-3.4.13/
export PATH=$PATH:$ZOOKEEPER_HOME/bin

• 运行: zkServer.sh start
• 查看状态： zkServer.sh status

二、安装Hadoop

1、编辑 hadoop-env.sh

主要设置为你自己的JAVA_HOME

export JAVA_HOME=/usr/local/java

2、编辑 core-site.xml

<configuration>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>node3:2181,node4:2181,node5:2181</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/hadoop/hadoop-data/temp</value>
    </property>

    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://leo</value>
    </property>
    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
    </property>
    <!-- Hue WebHDFS proxy user setting -->

    <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.logfile.size</name>
        <value>10000000</value>
        <description>The max size of each log file</description>
    </property>

    <property>
        <name>hadoop.logfile.count</name>
        <value>10</value>
        <description>The max number of log files</description>
    </property>

    <property>
        <name>ipc.client.connect.max.retries</name>
        <value>100</value>
    </property>
    <property>
        <name>ipc.client.connect.retry.interval</name>
        <value>10000</value>
    </property>
</configuration>

3、编辑 hdfs-site.xml

<configuration>
    <property>
        <name>dfs.nameservices</name>
        <value>leo</value>
    </property>
    <property>
        <name>dfs.ha.namenodes.leo</name>
        <value>nn1,nn2</value>
    </property>
    <property>
        <name>dfs.namenode.rpc-address.leo.nn1</name>
        <value>node1:8020</value>
    </property>
    <property>
        <name>dfs.namenode.rpc-address.leo.nn2</name>
        <value>node2:8020</value>
    </property>
    <property>
        <name>dfs.namenode.http-address.leo.nn1</name>
        <value>node1:50070</value>
    </property>
    <property>
        <name>dfs.namenode.http-address.leo.nn2</name>
        <value>node2:50070</value>
    </property>
    <property>
        <name>dfs.namenode.shared.edits.dir</name>
        <value>qjournal://node3:8485;node4:8485;node5:8485/leo</value>
    </property>
    <property>
        <name>dfs.client.failover.proxy.provider.leo</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    </property>
    <property>
        <name>dfs.ha.fencing.methods</name>
        <value>sshfence</value>
    </property>
    <property>
        <name>dfs.ha.fencing.ssh.private-key-files</name>
        <value>/home/hadoop/root/.ssh/id_rsa</value>
    </property>
    <property>
        <name>dfs.journalnode.edits.dir</name>
        <value>/home/hadoop/hadoop-data/journal</value>
    </property>
    <property>
        <name>dfs.ha.automatic-failover.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/home/hadoop/hadoop-data/data-node-data</value>
    </property>

    <property>
        <name>dfs.heartbeat.interval</name>
        <value>10</value>
    </property>
    <property>
        <name>dfs.qjournal.start-segment.timeout.ms</name>
        <value>60000</value>
    </property>
    <property>
        <name>dfs.qjournal.select-input-streams.timeout.ms</name>
        <value>60000</value>
    </property>
    <property>
        <name>dfs.qjournal.write-txns.timeout.ms</name>
        <value>60000</value>
    </property>
    <property>
        <name>dfs.datanode.max.transfer.threads</name>
        <value>163</value>
    </property>

    <property>
        <name>dfs.balance.bandwidthPerSec</name>
        <value>10485760</value>
        <description> Specifies the maximum bandwidth that each datanode can utilize for the balancing purpose in term of the number of bytes per second. </description>
    </property>
   <property>
        <name>dfs.permissions.enabled</name>
        <value>false</value>
    </property>
</configuration>

4、编辑 mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

    <property>
        <name>mapred.child.java.opts</name>
        <value>-Xmx1024m</value>
    </property>
    <property>
        <name>mapreduce.tasktracker.reduce.tasks.maximum</name>
        <value>5</value>
    </property>
        <property>
        <name>mapreduce.tasktracker.map.tasks.maximum</name>
        <value>5</value>
    </property>
</configuration>

5、编辑 yarn-site.xml

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>
    <property>
        <name>yarn.nodemanager.log-dirs</name>
        <value>/home/hadoop/log/yarn</value>
    </property>
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>false</value>
    </property>
    <property>
        <name>yarn.nodemanager.log.retain-seconds</name>
        <value>259200</value>
    </property>
    <property>
        <name>yarn.nodemanager.resource.cpu-vcores</name>
        <value>6</value>
    </property>
    <property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>false</value>
    </property>
    <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
    </property>

    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>10240</value>
    </property>
    <property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>51200</value>
    </property>

    <property>
        <name>yarn.scheduler.maximum-allocation-vcores</name>
        <value>30</value>
    </property>

    <!--<property>
        <name>yarn.resourcemanager.scheduler.class</name>
        <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
    </property>-->
    <!-- 开启RM高可靠 -->
    <property>
        <name>yarn.resourcemanager.ha.enabled</name>
        <value>true</value>
    </property>
    <!-- 指定RM的cluster id -->
    <property>
        <name>yarn.resourcemanager.cluster-id</name>
        <value>cluster1</value>
    </property>
    <!-- 指定RM的名字 -->
    <property>
        <name>yarn.resourcemanager.ha.rm-ids</name>
        <value>rm1,rm2</value>
    </property>
    <!-- 分别指定RM的地址 -->
    <property>
        <name>yarn.resourcemanager.hostname.rm1</name>
        <value>node4</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname.rm2</name>
        <value>node5</value>
    </property>

    <property>
        <name>yarn.resourcemanager.webapp.address.rm1 </name>
        <value>node4:8088</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address.rm2</name>
        <value>node5:8088</value>
    </property>
    <!-- 指定zk集群地址 -->
    <property>
        <name>yarn.resourcemanager.zk-address</name>
        <value>node3:2181,node4:2181,node5:2181</value>
    </property>

    <!-- YARN的NodeManager进行配置，使其支持Spark的Shuffle Service-->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle,spark_shuffle</value>
    </property>

    <property>
        <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
        <value>org.apache.spark.network.yarn.YarnShuffleService</value>
    </property>
</configuration>

6、编辑slaves文件

node1
node2
node3
node4
node5

7、分发hadoop到其余节点

8、增加hadoop的环境变量

export HADOOP_HOME=/home/hadoop/hadoop-2.7.4
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

9、启动zookeeper集群

分别在node3、node4、node5节点上执行：zkServer.sh start

10、启动journalnode

分别在node3、node4、node5节点上执行：hadoop-daemon.sh start journalnode

jps 命令检验，三个节点上多了JournalNode进程

11、格式化namenode

在node1节点上执行命令：hdfs namenode -format

启动node1的namenode：hadoop-deamon.sh start namenode

12、在node2节点上同步node1的namenode元数据

hdfs namenode -bootstrapStandby

启动node2节点的namenode: hadoop-deamon.sh start namenode

13、格式化zkfc

node1节点上执行：hdfs zkfc -formatZK

14、启动HDFS

node1节点上执行：start-dfs.sh

15、查看namenode状态

[hadoop@node1 hadoop]$ hdfs haadmin -getServiceState nn1
active
[hadoop@node1 hadoop]$ hdfs haadmin -getServiceState nn2
standby
[hadoop@node1 hadoop]$ 

16、启动yarn

node1节点上执行：start-yarn.sh

分别在node4、node5节点上启动resourcemanager: yarn-daemon.sh start resourcemanager

17、查看yarn状态

[hadoop@node1 hadoop]$ yarn rmadmin -getServiceState rm1
active
[hadoop@node1 hadoop]$ yarn rmadmin -getServiceState rm2
standby
[hadoop@node1 hadoop]$ 

18、查看网页