Hadoop部署(HA)(版本3.3.6)

一.前言

hadoop高可用集群分为 主备的namenode resourcemanager 数据同步监听的journalnode 和zkfc监控 部署规划如下

jdk版本选用 jdk-11.0.24 zookeeper版本选用3.6.3

二. 部署过程

2.1 jdk部署

#下载jdk11.0.24安装包 解压 创建软连接
tar -zxvf jdk-11.0.24_linux-x64_bin.tar.gz
ln -s /home/user1/jars/jdk-11.0.24/ jdk
#修改环境变量
vi .bash_profile
#增加
#JAVA_HOME
export JAVA_HOME=/home/user1/jdk
export PATH=$PATH:$JAVA_HOME/bin
#验证环境 显示java版本是否为java version 11
java -version

2.2 hadoop部署

#下载hadoop3.3.6安装包 解压 创建软连接
tar -zxvf hadoop-3.3.6.tar.gz
ln -s /home/user1/jars/hadoop-3.3.6/ /home/user1/hadoop
#修改环境变量
#HADOOP_HOME
export HADOOP_HOME=/opt/module/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin

2.3 配置文件修改

2.3.1 core-site.xml

<!-- Namenode自定义集群名称 -->
   <property>
     <name>fs.defaultFS</name>
        <value>hdfs://mycluster</value>
    </property>
<!-- 指定 hadoop 数据的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/user1/hadoop/data</value>
    </property>
<!-- 配置 HDFS 网页登录使用的静态用户为 user1 -->
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>user1</value>
    </property>
<!--配置超级代理-->
    <property>
        <name>hadoop.proxyuser.user1.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.user1.groups</name>
        <value>*</value>
    </property>
<!-- 配置ZKFC进程连接zookeeper的地址 -->
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>linux1:2181,linux2:2181,linux3:2181</value>
    </property>

2.3.2 hdfs-site.xml

 <property>
     <name>dfs.nameservices</name>
     <value>mycluster</value>
  </property>
<!-- namenode服务mycluster下3个节点 -->
  <property>
      <name>dfs.ha.namenodes.mycluster</name>
      <value>nn1,nn2</value>
  </property>
<!-- 节点通讯地址 -->
  <property>
      <name>dfs.namenode.rpc-address.mycluster.nn1</name>
      <value>linux1:8020</value>
  </property>
  <property>
      <name>dfs.namenode.rpc-address.mycluster.nn2</name>
      <value>linux2:8020</value>
  </property>
<!-- web ui地址 -->
  <property>
      <name>dfs.namenode.http-address.mycluster.nn1</name>
      <value>linux1:9870</value>
  </property>
  <property>
      <name>dfs.namenode.http-address.mycluster.nn2</name>
      <value>linux2:9870</value>
  </property>
 <!-- journalnode edits读取写入地址 -->
  <property>
      <name>dfs.namenode.shared.edits.dir</name>
      <value>qjournal://linux1:8485;linux2:8485/mycluster</value>
  </property>
<!--  the Java class that HDFS clients use to contact the Active NameNode -->
  <property>
      <name>dfs.client.failover.proxy.provider.mycluster</name>
      <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
  </property>
<!-- a list of scripts or Java classes which will be used to fence the Active NameNode during a
 failover -->
  <property>
      <name>dfs.ha.fencing.methods</name>
      <value>sshfence</value>
      <value>shell(/bin/true)</value>
  </property>
  <property>
      <name>dfs.ha.fencing.ssh.private-key-files</name>
      <value>/home/user1/.ssh/id_rsa</value>
  </property>
  <property>
      <name>dfs.ha.nn.not-become-active-in-safemode</name>
      <value>true</value>
  </property>
<!-- 故障情况自动切换 -->
  <property>
      <name>dfs.ha.automatic-failover.enabled</name>
      <value>true</value>
  </property>
<!-- Namenode 数据存储目录-->
  <property>
      <name>dfs.namenode.name.dir</name>
      <value>${hadoop.tmp.dir}/name</value>
  </property>
<!-- Datanode 数据存储目录-->
  <property>
      <name>dfs.namenode.data.dir</name>
      <value>${hadoop.tmp.dir}/data</value>
  </property>
<!-- journalnode 数据存储目录-->
  <property>
      <name>dfs.journalnode.edits.dir</name>
      <value>${hadoop.tmp.dir}/jn</value>
  </property>

2.3.3 yarn-site.xml

<property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>    
    <!--开启ResourceManager HA功能-->
    <property>
        <name>yarn.resourcemanager.ha.enabled</name>
        <value>true</value>
    </property>
    <!-- 集群的Id，使用该值确保RM不会做为其它集群的active -->
    <!--标志ResourceManager-->
    <property>
        <name>yarn.resourcemanager.cluster-id</name>
        <value>yarn-cluster</value>
    </property>
  
    <!--集群中ResourceManager的ID列表，后面的配置将引用该ID-->
    <property>
        <name>yarn.resourcemanager.ha.rm-ids</name>
        <value>rm1,rm2</value>
    </property>
  
    <!-- 设置YARN集群主角色运行节点rm1-->
    <property>
        <name>yarn.resourcemanager.hostname.rm1</name>
        <value>linux1</value>
    </property>
  
    <!-- 设置YARN集群主角色运行节点rm2-->
    <property>
        <name>yarn.resourcemanager.hostname.rm2</name>
        <value>linux2</value>
    </property>
  
    <!--ResourceManager1的Web页面访问地址-->
    <property>
        <name>yarn.resourcemanager.webapp.address.rm1</name>
        <value>linux1:8088</value>
    </property>

    <!--ResourceManager2的Web页面访问地址-->
    <property>
        <name>yarn.resourcemanager.webapp.address.rm2</name>
        <value>linux2:8088</value>
    </property>
    <!-- 配置第一台机器的resourceManager通信地址 -->
    <property>
         <name>yarn.resourcemanager.address.rm1</name>
         <value>linux1:8032</value>
    </property>
    <property>
         <name>yarn.resourcemanager.scheduler.address.rm1</name>
         <value>linux1:8030</value>
    </property>
    <property>
         <name>yarn.resourcemanager.resource-tracker.address.rm1</name>
         <value>linux1:8031</value>
    </property>
    <property>
         <name>yarn.resourcemanager.admin.address.rm1</name>
         <value>linux1:8033</value>
    </property>
    <!-- 配置第二台机器的resourceManager通信地址 -->
    <property>
         <name>yarn.resourcemanager.address.rm2</name>
         <value>linux2:8032</value>
    </property>
    <property>
         <name>yarn.resourcemanager.scheduler.address.rm2</name>
         <value>linux2:8030</value>
    </property>
    <property>
         <name>yarn.resourcemanager.resource-tracker.address.rm2</name>
         <value>linux2:8031</value>
    </property>
    <property>
         <name>yarn.resourcemanager.admin.address.rm2</name>
         <value>linux2:8033</value>
    </property>
    <!--启用ResouerceManager重启的功能，默认为false-->
    <property>
        <name>yarn.resourcemanager.recovery.enabled</name>
        <value>true</value>
    </property>
    <!--用于ResouerceManager状态存储的类-->
    <property>
        <name>yarn.resourcemanager.store.class</name>
        <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
    </property>
    <!--ZooKeeper集群列表-->
    <property>
        <name>yarn.resourcemanager.zk-address</name>
        <value>linux1:2181,linux2:2181,linux3:2181</value>
        <description>For multiple zk services, separate them with comma</description>
    </property>
    <!--开启resourcemanager故障自动切换，指定机器--> 
    <property>
        <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
        <value>true</value>
        <description>Enable automatic failover; By default, it is enabled only when HA is enabled.</description>
    </property>    
    <property>
        <name>yarn.client.failover-proxy-provider</name>
        <value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value>
    </property>
    <!-- 允许分配给一个任务最大的CPU核数，默认是8 -->
    <property>
        <name>yarn.nodemanager.resource.cpu-vcores</name>
        <value>4</value>
    </property>
    <!-- 每个节点可用内存,单位MB -->
    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>3072</value>
    </property>
    <!-- 单个任务可申请最少内存，默认1024MB -->
    <property>
         <name>yarn.scheduler.minimum-allocation-mb</name>
        <value>1024</value>
    </property>
    <!-- 单个任务可申请最大内存，默认8192MB -->
    <property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>4096</value>
    </property>
    <!--多长时间聚合删除一次日志 此处-->
    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>2592000</value><!--30 day-->
    </property>
    <!--时间在几秒钟内保留用户日志。只适用于如果日志聚合是禁用的-->
    <property>
        <name>yarn.nodemanager.log.retain-seconds</name>
        <value>604800</value><!--7 day-->
    </property>
    <!--指定文件压缩类型用于压缩汇总日志-->
    <property>
        <name>yarn.nodemanager.log-aggregation.compression-type</name>
        <value>gz</value>
    </property>
    <!-- nodemanager本地文件存储目录-->
    <property>
        <name>yarn.nodemanager.local-dirs</name>
        <value>/home/user1/hadoop/data/local</value>
    </property>
    <!-- resourceManager  保存最大的任务完成个数 -->
    <property>
        <name>yarn.resourcemanager.max-completed-applications</name>
        <value>1000</value>
    </property>
    <!-- 逗号隔开的服务列表，列表名称应该只包含a-zA-Z0-9_,不能以数字开始-->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <!-- 是否将对容器实施虚拟内存限制 -->
    <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
    </property>
    <!-- 设置日志聚集服务器地址 -->
    <property>
        <name>yarn.log.server.url</name>
        <value>http://linux1:19888/jobhistory/logs</value>
    </property>
    <!--环境变量通过从NodeManagers的容器继承的环境属性，对于MapReduce应用程序，除了 默认值 hadoop op_mapred_home 应被加入 外，还有如下属性值：-->
     <property>
         <name>yarn.nodemanager.env-whitelist</name>
         <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
     </property>
    <property>
        <name>yarn.application.classpath</name>
        <value>
        /home/user1/hadoop/etc/hadoop,
        /home/user1/hadoop/share/hadoop/common/*,
        /home/user1/hadoop/share/hadoop/common/lib/*,
        /home/user1/hadoop/share/hadoop/hdfs/*,
        /home/user1/hadoop/share/hadoop/hdfs/lib/*,
        /home/user1/hadoop/share/hadoop/mapreduce/*,
        /home/user1/hadoop/share/hadoop/mapreduce/lib/*,
        /home/user1/hadoop/share/hadoop/yarn/*,
        /home/user1/hadoop/share/hadoop/yarn/lib/*
       </value>
</property>

2.3.4 mapred-site.xml

<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
    <description>MapReduce的运行框架设置为YARN</description>
</property>

<property>
    <name>mapreduce.jobhistory.address</name>
    <value>linux1:10020</value>
    <description>历史服务器通讯端口为master：10020</description>
</property>
<property>
    <name>mapreduce.jobhistory.intermediate-done-dir</name>
    <value>/input/tmp</value>
    <description>历史信息在HDFS的记录临时路径</description>
</property>

<property>
     <name>mapreduce.jobhistory.done-dir</name>
     <value>/input/done</value>
     <description>历史信息在HDFS的记录路径</description>
</property>
<property>
     <name>yarn.app.mapreduce.am.env</name>
     <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
     <description>MAPREDUCE HOME 设置为HADOOP_HOME</description>
</property>

<property>
    <name>mapreduce.map.env</name>
    <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    <description>MAPREDUCE HOME 设置为HADOOP_HOME</description>
</property>
<property>
    <name>mapreduce.reduce.env</name>
    <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    <description>MAPREDUCE HOME 设置为HADOOP_HOME</description>
</property>
<property>
    <name>yarn.app.mapreduce.am.resource.mb</name>
    <value>1024</value>
</property>
<property>
    <name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
    <value>1</value>
</property>

2.3.5 workers

linux1
linux2
linux3
linux4
linux5

2.4 启动 hdfs

#启动journalnode 在有namenode的节点执行
hdfs --daemon start journalnode
#namenode格式化 (只需第一台节点执行)
hdfs namenode -format
#启动namenode 两台节点分别执行
hdfs --daemon start namenode
#格式化zkfc(在所有zkfc的节点)
hdfs zkfc -formatZK
#启动zkfc
hdfs --daemon start zkfc
#依次启动datanode
hdfs --daemon start datanode

2.4.1 hdfs启动验证

#打开web界面 查看界面是否可以打开datanode在线情况 端口9870
#命令行查看节点状态是否为一主一备
hdfs haadmin -getAllServiceState
#上传文件测试
hdfs dfs -put /home/user1/a.txt /input

2.5 启动yarn

#启动resourcemanager
yarn --daemon start resourcemanager
#启动nodemanager
yarn --daemon start nodemanager
#打开web界面查看节点在线个数 端口8088
#yarn运行任务测试
yarn jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar wordcount /input/a.txt /output
#查看节点状态
yarn node -list

三.遇到的问题

3.1 问题 : 主机命令行变成 -bash-4.2$

#解决 :切换到有故障的用户 su - user1
#2.复制/etc/skel 里面的模板到当前用户
cp /etc/skel/.bash* ~
#3.退出重新登录用户(环境变量丢失需要重新配)

3.2 yarn任务启动报错 : Could not find or load main class org.apache.hadoop.mapred.YarnChild 或者 org.apache.hadoop.mapreduce.v2.app.MRAppMaster

#在yarn-site.xml中添加

    <property>
        <name>yarn.application.classpath</name>
        <value>
        /home/user1/hadoop/etc/hadoop,
        /home/user1/hadoop/share/hadoop/common/*,
        /home/user1/hadoop/share/hadoop/common/lib/*,
        /home/user1/hadoop/share/hadoop/hdfs/*,
        /home/user1/hadoop/share/hadoop/hdfs/lib/*,
        /home/user1/hadoop/share/hadoop/mapreduce/*,
        /home/user1/hadoop/share/hadoop/mapreduce/lib/*,
        /home/user1/hadoop/share/hadoop/yarn/*,
        /home/user1/hadoop/share/hadoop/yarn/lib/*
       </value>
</property>

3.3 Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException): Invalid resource request! Cannot allocate containers as requested resource is greater than maximum allowed allocation. Requested resource type=[memory-mb], Requested resource=<memory:1536, vCores:1>, maximum allowed allocation=<memory:1024, vCores:2>, please note that maximum allowed allocation is calculated by scheduler based on maximum resource of registered NodeManagers, which might be less than configured maximum allocation=<memory:8192, vCores:4>

<property>
        <name>yarn.app.mapreduce.am.resource.mb</name>
        <value>1024</value>
</property>
<property>
        <name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
        <value>1</value>
</property>

#这是因为nodemanager的配置内存是1024m,但是mr默认的最小内存是1536 除了设置此mr任务作业最小内存外,还可以通过修改yarn-site增大资源配置解决

3.4 还有一些其他问题 诸如 两个namenode都是standby(此问题为初始化有问题,重新初始化启动) 主机重启后启动namenode一直报错文件权限有问题(网上说是linux重启丢失hadoop的tmp文件的权限,可以修改配置文件指定为/local目录下,或者删除数据目录重新初始化,生产不建议)等问题

四.常用命令

#4.1 hdfs
#查看目录结构
hdfs dfs -ls /
#创建目录
hdfs dfs -mkdir /test
#删除
hdfs dfs -rm   hdfs dfs -rm -r
#上传文件
hdfs dfs -put /文件路径 /目标路径
下载文件
hdfs dfs -get /下载路径 /本地路径
移动文件
hdfs dfs -mv /源路径 /目标路径
#4.2 yarn
#查看节点状态
yarn node -list
#查看任务列表
yarn application -list
#查看任务日志
yarn logs -applicationId
#查看队列信息
yarn queue -status
#杀掉任务
yarn application -kill
#提交任务
yarn jar