一.前言
hadoop高可用集群分为 主备的namenode resourcemanager 数据同步监听的journalnode 和zkfc监控 部署规划如下
jdk版本选用 jdk-11.0.24 zookeeper版本选用3.6.3
二. 部署过程
2.1 jdk部署
#下载jdk11.0.24安装包 解压 创建软连接
tar -zxvf jdk-11.0.24_linux-x64_bin.tar.gz
ln -s /home/user1/jars/jdk-11.0.24/ jdk
#修改环境变量
vi .bash_profile
#增加
#JAVA_HOME
export JAVA_HOME=/home/user1/jdk
export PATH=
P
A
T
H
:
PATH:
PATH:JAVA_HOME/bin
#验证环境 显示java版本是否为java version 11
java -version
2.2 hadoop部署
#下载hadoop3.3.6安装包 解压 创建软连接
tar -zxvf hadoop-3.3.6.tar.gz
ln -s /home/user1/jars/hadoop-3.3.6/ /home/user1/hadoop
#修改环境变量
#HADOOP_HOME
export HADOOP_HOME=/opt/module/hadoop
export PATH=
P
A
T
H
:
PATH:
PATH:HADOOP_HOME/bin
export PATH=
P
A
T
H
:
PATH:
PATH:HADOOP_HOME/sbin
2.3 配置文件修改
2.3.1 core-site.xml
<!-- Namenode自定义集群名称 -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster</value>
</property>
<!-- 指定 hadoop 数据的存储目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/home/user1/hadoop/data</value>
</property>
<!-- 配置 HDFS 网页登录使用的静态用户为 user1 -->
<property>
<name>hadoop.http.staticuser.user</name>
<value>user1</value>
</property>
<!--配置超级代理-->
<property>
<name>hadoop.proxyuser.user1.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.user1.groups</name>
<value>*</value>
</property>
<!-- 配置ZKFC进程连接zookeeper的地址 -->
<property>
<name>ha.zookeeper.quorum</name>
<value>linux1:2181,linux2:2181,linux3:2181</value>
</property>
2.3.2 hdfs-site.xml
<property>
<name>dfs.nameservices</name>
<value>mycluster</value>
</property>
<!-- namenode服务mycluster下3个节点 -->
<property>
<name>dfs.ha.namenodes.mycluster</name>
<value>nn1,nn2</value>
</property>
<!-- 节点通讯地址 -->
<property>
<name>dfs.namenode.rpc-address.mycluster.nn1</name>
<value>linux1:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn2</name>
<value>linux2:8020</value>
</property>
<!-- web ui地址 -->
<property>
<name>dfs.namenode.http-address.mycluster.nn1</name>
<value>linux1:9870</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn2</name>
<value>linux2:9870</value>
</property>
<!-- journalnode edits读取写入地址 -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://linux1:8485;linux2:8485/mycluster</value>
</property>
<!-- the Java class that HDFS clients use to contact the Active NameNode -->
<property>
<name>dfs.client.failover.proxy.provider.mycluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- a list of scripts or Java classes which will be used to fence the Active NameNode during a
failover -->
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
<value>shell(/bin/true)</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/user1/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.ha.nn.not-become-active-in-safemode</name>
<value>true</value>
</property>
<!-- 故障情况自动切换 -->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!-- Namenode 数据存储目录-->
<property>
<name>dfs.namenode.name.dir</name>
<value>${hadoop.tmp.dir}/name</value>
</property>
<!-- Datanode 数据存储目录-->
<property>
<name>dfs.namenode.data.dir</name>
<value>${hadoop.tmp.dir}/data</value>
</property>
<!-- journalnode 数据存储目录-->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>${hadoop.tmp.dir}/jn</value>
</property>
2.3.3 yarn-site.xml
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<!--开启ResourceManager HA功能-->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<!-- 集群的Id,使用该值确保RM不会做为其它集群的active -->
<!--标志ResourceManager-->
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>yarn-cluster</value>
</property>
<!--集群中ResourceManager的ID列表,后面的配置将引用该ID-->
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<!-- 设置YARN集群主角色运行节点rm1-->
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>linux1</value>
</property>
<!-- 设置YARN集群主角色运行节点rm2-->
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>linux2</value>
</property>
<!--ResourceManager1的Web页面访问地址-->
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>linux1:8088</value>
</property>
<!--ResourceManager2的Web页面访问地址-->
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>linux2:8088</value>
</property>
<!-- 配置第一台机器的resourceManager通信地址 -->
<property>
<name>yarn.resourcemanager.address.rm1</name>
<value>linux1:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm1</name>
<value>linux1:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm1</name>
<value>linux1:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm1</name>
<value>linux1:8033</value>
</property>
<!-- 配置第二台机器的resourceManager通信地址 -->
<property>
<name>yarn.resourcemanager.address.rm2</name>
<value>linux2:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm2</name>
<value>linux2:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm2</name>
<value>linux2:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm2</name>
<value>linux2:8033</value>
</property>
<!--启用ResouerceManager重启的功能,默认为false-->
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<!--用于ResouerceManager状态存储的类-->
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
<!--ZooKeeper集群列表-->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>linux1:2181,linux2:2181,linux3:2181</value>
<description>For multiple zk services, separate them with comma</description>
</property>
<!--开启resourcemanager故障自动切换,指定机器-->
<property>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
<value>true</value>
<description>Enable automatic failover; By default, it is enabled only when HA is enabled.</description>
</property>
<property>
<name>yarn.client.failover-proxy-provider</name>
<value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value>
</property>
<!-- 允许分配给一个任务最大的CPU核数,默认是8 -->
<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>4</value>
</property>
<!-- 每个节点可用内存,单位MB -->
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>3072</value>
</property>
<!-- 单个任务可申请最少内存,默认1024MB -->
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>1024</value>
</property>
<!-- 单个任务可申请最大内存,默认8192MB -->
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>4096</value>
</property>
<!--多长时间聚合删除一次日志 此处-->
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>2592000</value><!--30 day-->
</property>
<!--时间在几秒钟内保留用户日志。只适用于如果日志聚合是禁用的-->
<property>
<name>yarn.nodemanager.log.retain-seconds</name>
<value>604800</value><!--7 day-->
</property>
<!--指定文件压缩类型用于压缩汇总日志-->
<property>
<name>yarn.nodemanager.log-aggregation.compression-type</name>
<value>gz</value>
</property>
<!-- nodemanager本地文件存储目录-->
<property>
<name>yarn.nodemanager.local-dirs</name>
<value>/home/user1/hadoop/data/local</value>
</property>
<!-- resourceManager 保存最大的任务完成个数 -->
<property>
<name>yarn.resourcemanager.max-completed-applications</name>
<value>1000</value>
</property>
<!-- 逗号隔开的服务列表,列表名称应该只包含a-zA-Z0-9_,不能以数字开始-->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<!-- 是否将对容器实施虚拟内存限制 -->
<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>
<!-- 设置日志聚集服务器地址 -->
<property>
<name>yarn.log.server.url</name>
<value>http://linux1:19888/jobhistory/logs</value>
</property>
<!--环境变量通过从NodeManagers的容器继承的环境属性,对于MapReduce应用程序,除了 默认值 hadoop op_mapred_home 应被加入 外,还有如下属性值:-->
<property>
<name>yarn.nodemanager.env-whitelist</name>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
</property>
<property>
<name>yarn.application.classpath</name>
<value>
/home/user1/hadoop/etc/hadoop,
/home/user1/hadoop/share/hadoop/common/*,
/home/user1/hadoop/share/hadoop/common/lib/*,
/home/user1/hadoop/share/hadoop/hdfs/*,
/home/user1/hadoop/share/hadoop/hdfs/lib/*,
/home/user1/hadoop/share/hadoop/mapreduce/*,
/home/user1/hadoop/share/hadoop/mapreduce/lib/*,
/home/user1/hadoop/share/hadoop/yarn/*,
/home/user1/hadoop/share/hadoop/yarn/lib/*
</value>
</property>
2.3.4 mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
<description>MapReduce的运行框架设置为YARN</description>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>linux1:10020</value>
<description>历史服务器通讯端口为master:10020</description>
</property>
<property>
<name>mapreduce.jobhistory.intermediate-done-dir</name>
<value>/input/tmp</value>
<description>历史信息在HDFS的记录临时路径</description>
</property>
<property>
<name>mapreduce.jobhistory.done-dir</name>
<value>/input/done</value>
<description>历史信息在HDFS的记录路径</description>
</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
<description>MAPREDUCE HOME 设置为HADOOP_HOME</description>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
<description>MAPREDUCE HOME 设置为HADOOP_HOME</description>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
<description>MAPREDUCE HOME 设置为HADOOP_HOME</description>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
<value>1024</value>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
<value>1</value>
</property>
2.3.5 workers
linux1
linux2
linux3
linux4
linux5
2.4 启动 hdfs
#启动journalnode 在有namenode的节点执行
hdfs --daemon start journalnode
#namenode格式化 (只需第一台节点执行)
hdfs namenode -format
#启动namenode 两台节点分别执行
hdfs --daemon start namenode
#格式化zkfc(在所有zkfc的节点)
hdfs zkfc -formatZK
#启动zkfc
hdfs --daemon start zkfc
#依次启动datanode
hdfs --daemon start datanode
2.4.1 hdfs启动验证
#打开web界面 查看界面是否可以打开datanode在线情况 端口9870
#命令行查看节点状态是否为一主一备
hdfs haadmin -getAllServiceState
#上传文件测试
hdfs dfs -put /home/user1/a.txt /input
2.5 启动yarn
#启动resourcemanager
yarn --daemon start resourcemanager
#启动nodemanager
yarn --daemon start nodemanager
#打开web界面查看节点在线个数 端口8088
#yarn运行任务测试
yarn jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar wordcount /input/a.txt /output
#查看节点状态
yarn node -list
三.遇到的问题
3.1 问题 : 主机命令行变成 -bash-4.2$
#解决 :切换到有故障的用户 su - user1
#2.复制/etc/skel 里面的模板到当前用户
cp /etc/skel/.bash* ~
#3.退出重新登录用户(环境变量丢失需要重新配)
3.2 yarn任务启动报错 : Could not find or load main class org.apache.hadoop.mapred.YarnChild 或者 org.apache.hadoop.mapreduce.v2.app.MRAppMaster
#在yarn-site.xml中添加
<property>
<name>yarn.application.classpath</name>
<value>
/home/user1/hadoop/etc/hadoop,
/home/user1/hadoop/share/hadoop/common/*,
/home/user1/hadoop/share/hadoop/common/lib/*,
/home/user1/hadoop/share/hadoop/hdfs/*,
/home/user1/hadoop/share/hadoop/hdfs/lib/*,
/home/user1/hadoop/share/hadoop/mapreduce/*,
/home/user1/hadoop/share/hadoop/mapreduce/lib/*,
/home/user1/hadoop/share/hadoop/yarn/*,
/home/user1/hadoop/share/hadoop/yarn/lib/*
</value>
</property>
3.3 Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException): Invalid resource request! Cannot allocate containers as requested resource is greater than maximum allowed allocation. Requested resource type=[memory-mb], Requested resource=<memory:1536, vCores:1>, maximum allowed allocation=<memory:1024, vCores:2>, please note that maximum allowed allocation is calculated by scheduler based on maximum resource of registered NodeManagers, which might be less than configured maximum allocation=<memory:8192, vCores:4>
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
<value>1024</value>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
<value>1</value>
</property>
#这是因为nodemanager的配置内存是1024m,但是mr默认的最小内存是1536 除了设置此mr任务作业最小内存外,还可以通过修改yarn-site增大资源配置解决
3.4 还有一些其他问题 诸如 两个namenode都是standby(此问题为初始化有问题,重新初始化启动) 主机重启后启动namenode一直报错文件权限有问题(网上说是linux重启丢失hadoop的tmp文件的权限,可以修改配置文件指定为/local目录下,或者删除数据目录重新初始化,生产不建议)等问题
四.常用命令
#4.1 hdfs
#查看目录结构
hdfs dfs -ls /
#创建目录
hdfs dfs -mkdir /test
#删除
hdfs dfs -rm hdfs dfs -rm -r
#上传文件
hdfs dfs -put /文件路径 /目标路径
下载文件
hdfs dfs -get /下载路径 /本地路径
移动文件
hdfs dfs -mv /源路径 /目标路径
#4.2 yarn
#查看节点状态
yarn node -list
#查看任务列表
yarn application -list
#查看任务日志
yarn logs -applicationId
#查看队列信息
yarn queue -status
#杀掉任务
yarn application -kill
#提交任务
yarn jar