hadoop 分布式集群部署一 YARN 高可用部署

CodingDemo

已于 2023-08-10 19:26:01 修改

阅读量402

点赞数

分类专栏：大数据文章标签：分布式 hadoop 大数据

于 2023-08-10 14:36:37 首次发布

本文链接：https://blog.csdn.net/yxydde/article/details/132208174

版权

大数据专栏收录该内容

4 篇文章 0 订阅

订阅专栏

1、环境准备和基础配置

hadoop 分布式集群部署一 zookeeper 集群部署
 HDFS 高可用部署（Observer NameNode 进行读写分离）

2、创建YARN用户和相关目录

# 在所有运行yarn相关服务的节点上创建 yarn 用户
~]$ useradd yarn
# 在 mr-history 节点上创建 mapred 用户
~]$ useradd mapred

# nodemanager 节点假如有4块数据盘，分别挂在 /data/1 /data/2 /data/3 /data/4 目录下
~]# mkdir -pv /data/{1,2,3,4}/yarn
~]# chown -R yarn:yarn /data/{1,2,3,4}/yarn

3、配置 yarn-site.xml

~]$ vi /etc/hadoop/conf/yarn-site.xml 
<?xml version="1.0"?>
<configuration>

        <!-- Generated by Jinja template -->

        <property>
                <name>yarn.resourcemanager.ha.enabled</name>
                <value>true</value>
        </property>
        <property>
                <name>yarn.resourcemanager.cluster-id</name>
                <value>cluster</value>
        </property>
        <property>
                <name>yarn.resourcemanager.ha.rm-ids</name>
                <value>rm1,rm2</value>
        </property>
        <property>
                <name>yarn.resourcemanager.recovery.enabled</name>
                <value>true"</value>
        </property>
        <property>
                <name>yarn.resourcemanager.store.class</name>
                <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
        </property>
        <property>
                <name>hadoop.zk.address</name>
                <value>192.168.36.131:2181,192.168.36.132:2181,192.168.36.133:2181</value>
        </property>
        <property>
                <name>yarn.nodemanager.resource.cpu-vcores</name>
                <value>8</value>
        </property>
        <property>
                <name>yarn.nodemanager.resource.memory-mb</name>
                <value>16384</value>
        </property>
        <property>
                <name>yarn.log-aggregation-enable</name>
                <value>true</value>
        </property>
        <property>
                <name>yarn.log-aggregation.retain-seconds</name>
                <value>604800</value>
        </property>
        <property>
                <name>yarn.log-aggregation.retain-check-interval-seconds</name>
                <value>86400</value>
        </property>
        <property>
                <name>yarn.nodemanager.log-dirs</name>
                <value>/data/1/yarn/logs,/data/2/yarn/logs,/data/3/yarn/logs,/data/4/yarn/logs</value>
        </property>
        <property>
                <name>yarn.nodemanager.local-dirs</name>
                <value>/data/1/yarn/local,/data/2/yarn/local,/data/3/yarn/local,/data/4/yarn/local</value>
        </property>
        <property>
                <name>yarn.nodemanager.remote-app-log-dir</name>
                <value>/tmp/logs</value>
        </property>
        <property>
                <name>yarn.log.server.url</name>
                <value>http://192.168.36.137:19888/jobhistory/logs</value>
        </property>
        <property>
                <name>yarn.scheduler.maximum-allocation-vcores</name>
                <value>8</value>
        </property>
        <property>
                <name>yarn.scheduler.maximum-allocation-mb</name>
                <value>16384</value>
        </property>
        <property>
                <name>yarn.resourcemanager.hostname.rm1</name>
                <value>192.168.36.131</value>
        </property>
        <property>
                <name>yarn.resourcemanager.webapp.address.rm1</name>
                <value>192.168.36.131:8088</value>
        </property>
        <property>
                <name>yarn.resourcemanager.hostname.rm2</name>
                <value>192.168.36.132</value>
        </property>
        <property>
                <name>yarn.resourcemanager.webapp.address.rm2</name>
                <value>192.168.36.132:8088</value>
        </property>

        <!-- Generated by Jinja template -->
</configuration>

4、配置 mapred-site.xml

]$ cat /etc/hadoop/conf/mapred-site.xml 
<?xml version="1.0"?>
<configuration>

        <!-- Generated by Jinja template -->

        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
        <property>
                <name>mapreduce.map.memory.mb</name>
                <value>1024</value>
        </property>
        <property>
                <name>mapreduce.reduce.memory.mb</name>
                <value>1024</value>
        </property>
        <property>
                <name>mapreduce.job.heap.memory-mb.ratio</name>
                <value>0.80</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.webapp.address</name>
                <value>192.168.36.137:19888</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.address</name>
                <value>192.168.36.137:10020</value>
        </property>
        <property>
                <name>mapreduce.map.env</name>
                <value>HADOOP_MAPRED_HOME=/opt/hadoop-3.3.6</value>
        </property>
        <property>
                <name>mapreduce.reduce.env</name>
                <value>HADOOP_MAPRED_HOME=/opt/hadoop-3.3.6</value>
        </property>
        <property>
                <name>yarn.app.mapreduce.am.env</name>
                <value>HADOOP_MAPRED_HOME=/opt/hadoop-3.3.6</value>
        </property>
        <property>
                <name>yarn.app.mapreduce.am.staging-dir</name>
                <value>/tmp/hadoop-yarn/staging</value>
        </property>
        <property>
                <name>yarn.app.mapreduce.am.resource.mb</name>
                <value>1024</value>
        </property>
        <property>
                <name>mapreduce.task.io.sort.mb</name>
                <value>512</value>
        </property>
        <property>
                <name>mapreduce.task.io.sort.factor</name>
                <value>100</value>
        </property>
        <property>
                <name>mapreduce.reduce.shuffle.parallelcopies</name>
                <value>50</value>
        </property>
        <property>
                <name>mapreduce.map.output.compress</name>
                <value>true</value>
        </property>
        <property>
                <name>mapreduce.map.output.compress.codec</name>
                <value>org.apache.hadoop.io.compress.SnappyCodec</value>
        </property>

        <!-- Generated by Jinja template -->
</configuration>

4、配置 yarn-env.sh 和 mapred-env.sh

~]$ vi /etc/hadoop/conf/yarn-env.sh
export HADOOP_LOG_DIR=/var/log/yarn
export HADOOP_PID_DIR=/opt/hadoop-3.3.6/run/yarn
 
~$ vi /etc/hadoop/conf/mapred-env.sh 
export HADOOP_LOG_DIR=/var/log/mapred
export HADOOP_PID_DIR=/opt/hadoop-3.3.6/run/mapred

5、注册并启动服务

# 在resourcemanager 节点注册并启动服务
~]$ cat /usr/lib/systemd/system/resourcemanager.service
[Unit]
Description=resourcemanager Service
After=network.target
[Service]
User=yarn
EnvironmentFile=-/etc/hadoop/conf/hadoop-env.sh
ExecStart=/opt/hadoop-3.3.6/bin/yarn --daemon start resourcemanager
PIDFile=/opt/hadoop-3.3.6/run/yarn/hadoop-yarn-resourcemanager.pid
Restart=on-failure
[Install]
WantedBy=multi-user.target

~]# systemctl enable --now resourcemanager

# 在nodemanager节点上注册并启动服务
~]# vi /usr/lib/systemd/system/nodemanager.service
[Unit]
Description=nodemanager Service
After=network.target
[Service]
User=yarn
EnvironmentFile=-/etc/hadoop/conf/hadoop-env.sh
ExecStart=/opt/hadoop-3.3.6/bin/yarn --daemon start nodemanager
PIDFile=/opt/hadoop-3.3.6/run/yarn/hadoop-yarn-nodemanager.pid
Restart=on-failure
[Install]
WantedBy=multi-user.target

~]# systemctl enable --now nodemanager

# 在 MR jobhistory 节点注册并启动服务
~]$ cat /usr/lib/systemd/system/mr-jobhistory.service
[Unit]
Description=mr-jobhistory Service
After=network.target
[Service]
User=mapred
Environment=HADOOP_PROXY_USER=hdfs
EnvironmentFile=-/etc/hadoop/conf/hadoop-env.sh
ExecStart=/opt/hadoop-3.3.6/bin/mapred --daemon start historyserver
PIDFile=/opt/hadoop-3.3.6/run/mapred/hadoop-mapred-historyserver.pid
Restart=on-failure
[Install]
WantedBy=multi-user.target

~]# systemctl enable --now mr-jobhistory

在这里插入图片描述