hadoop集群安装部署

最新推荐文章于 2024-06-12 20:13:34 发布

lql_h

最新推荐文章于 2024-06-12 20:13:34 发布

阅读量241

点赞数

分类专栏：大数据

本文链接：https://blog.csdn.net/lql_h/article/details/105111039

版权

大数据专栏收录该内容

19 篇文章 0 订阅

订阅专栏

HDFS

解压 Hadoop

tar -xzvf hadoop-2.6.0-cdh5.10.0.tar.gz

创建软连接

ln -s hadoop-2.6.0-cdh5.10.0 hadoop

修改配置文件

路径

/root/app/hadoop-2.6.0-cdh5.10.0/etc/hadoop

core-site.xml


<configuration>

        <property>

                <name>fs.defaultFS</name>

                <value>hdfs://mycluster</value>

        </property>

        <!--默认的HDFS路径-->

        <property>

                <name>hadoop.tmp.dir</name>

                <value>/root/data/tmp</value>

        </property>

        <!--hadoop的临时目录，如果需要配置多个目录，需要逗号隔开-->

        <property>

        <name>ha.zookeeper.quorum</name>

        <value>cdh1:2181,cdh2:2181,cdh3:2181</value>

        </property>

        <!--配置Zookeeper 管理HDFS-->

</configuration>

hdfs-site.xml


<configuration>

                <property>

                <name>dfs.replication</name>

                <value>3</value>

        </property>

                <!--数据块副本数为3-->

                <property>

                <name>dfs.permissions</name>

                <value>false</value>

        </property>

                <property>

                <name>dfs.permissions.enabled</name>

                <value>false</value>

        </property>

                <!--权限默认配置为false-->

                <property>

                <name>dfs.nameservices</name>

                <value>mycluster</value>

        </property>

                <!--命名空间，它的值与fs.defaultFS的值要对应，namenode高可用之后有两个namenode，mycluster是对外提供的统一入口-->

                <property>

                <name>dfs.ha.namenodes.mycluster</name>

                <value>nn1,nn2</value>

        </property>

                <!-- 指定 nameService 是 mycluster时的nameNode有哪些，这里的值也是逻辑名称，名字随便起，相互不重复即可-->

        <property>

                <name>dfs.namenode.rpc-address.mycluster.nn1</name>

                <value>cdh1:9000</value>

        </property>

        <property>

                <name>dfs.namenode.http-address.mycluster.nn1</name>

                <value>cdh1:50070</value>

        </property>

                <property>

                <name>dfs.namenode.rpc-address.mycluster.nn2</name>

                <value>cdh2:9000</value>

        </property>

        <property>

                <name>dfs.namenode.http-address.mycluster.nn2</name>

                <value>cdh2:50070</value>

        </property>

                <property>

                <name>dfs.ha.automatic-failover.enabled</name>

                <value>true</value>

        </property>

                <!--启动故障自动恢复-->

        <property>

                <name>dfs.namenode.shared.edits.dir</name>

                <value>qjournal://cdh1:8485;cdh2:8485;cdh3:8485/mycluster</value>

        </property>

                <!--指定NameNode的元数据在JournalNode上的存放位置-->

                <property>

                <name>dfs.client.failover.proxy.provider.mycluster</name>

                <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

        </property>

                <!--指定 mycluster 出故障时，哪个实现类负责执行故障切换-->

        <property>

                <name>dfs.journalnode.edits.dir</name>

                <value>/root/data/journaldata/jn</value>

        </property>

                <!-- 指定JournalNode在本地磁盘存放数据的位置 -->

                <property>

                <name>dfs.ha.fencing.methods</name>

                <value>shell(/bin/true)</value>

        </property>

                <!-- 配置隔离机制,shell通过ssh连接active namenode节点，杀掉进程-->

        <property>

                <name>dfs.ha.fencing.ssh.private-key-files</name>

                <value>/root/.ssh/id_rsa</value>

        </property>

                <!-- 为了实现SSH登录杀掉进程，还需要配置免密码登录的SSH密匙信息 -->

                <property>

                <name>dfs.ha.fencing.ssh.connect-timeout</name>

                <value>10000</value>

        </property>

        <property>

                <name>dfs.namenode.handler.count</name>

                <value>100</value>

        </property>

</configuration>

slaves


cdh1

cdh2

cdh3

hadoop-env.sh


export JAVA_HOME=/root/app/jdk

export HADOOP_HOME=/root/app/hadoop

yarn-site.xml

<configuration>

<property>

        <name>yarn.resourcemanager.connect.retry-interval.ms</name>

        <value>2000</value>

</property>

<property>

        <name>yarn.resourcemanager.ha.enabled</name>

        <value>true</value>

</property>

<!--打开高可用-->

<property>

        <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>

        <value>true</value>

</property>

<!--启动故障自动恢复-->

<property>

        <name>yarn.resourcemanager.ha.automatic-failover.embedded</name>

        <value>true</value>

</property>

<!--rm启动内置选举active-->

<property>

        <name>yarn.resourcemanager.cluster-id</name>

        <value>yarn-rm-cluster</value>

</property>

<!--给yarn cluster 取个名字yarn-rm-cluster-->

<property>

        <name>yarn.resourcemanager.ha.rm-ids</name>

        <value>rm1,rm2</value>

</property>

<!--ResourceManager高可用 rm1,rm2-->

<property>

        <name>yarn.resourcemanager.hostname.rm1</name>

        <value>cdh1</value>

</property>

<property>

        <name>yarn.resourcemanager.hostname.rm2</name>

        <value>cdh2</value>

</property>

<property>

        <name>yarn.resourcemanager.recovery.enabled</name>

        <value>true</value>

</property>

<!--启用resourcemanager 自动恢复-->

<property>

        <name>yarn.resourcemanager.zk.state-store.address</name>

        <value>cdh1:2181,cdh2:2181,cdh3:2181</value>

</property>

<!--状态存储地址-->

<property>

        <name>yarn.resourcemanager.zk-address</name>

        <value>cdh1:2181,cdh2:2181,cdh3:2181</value>

</property>

<!--配置Zookeeper地址-->

<property>

        <name>yarn.resourcemanager.address.rm1</name>

        <value>cdh1:8032</value>

</property>

<!--rm1端口号-->

<property>

        <name>yarn.resourcemanager.scheduler.address.rm1</name>

        <value>cdh1:8034</value>

</property>

<!-- rm1调度器的端口号-->

<property>

        <name>yarn.resourcemanager.webapp.address.rm1</name>

        <value>cdh1:8088</value>

</property>

<!-- rm1 webapp端口号-->

<property>

        <name>yarn.resourcemanager.address.rm2</name>

        <value>cdh2:8032</value>

</property>

<property>

        <name>yarn.resourcemanager.scheduler.address.rm2</name>

        <value>cdh2:8034</value>

</property>

<property>

        <name>yarn.resourcemanager.webapp.address.rm2</name>

        <value>cdh2:8088</value>

</property>

<property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

</property>

<property>

        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>

        <value>org.apache.hadoop.mapred.ShuffleHandler</value>

</property>

<!--执行MapReduce需要配置的shuffle过程-->

</configuration>

mapred-site


<configuration>

<property>

        <name>mapreduce.framework.name</name>

        <value>yarn</value>

</property>

<!--MapReduce以yarn模式运行-->

</configuration>

安装目录同步到其他节点

./deploy.sh /root/app/hadoop-2.6.0-cdh5.10.0 /root/app slave

各个节点分别创建软连接

ln -s hadoop-2.6.0-cdh5.10.0 hadoop

Hdfs格式化

./runRemoteCmd.sh "/root/app/zookeeper/bin/zkServer.sh start" all

./runRemoteCmd.sh "/root/app/hadoop/sbin/hadoop-daemon.sh start journalnode" all

[root@cdh1 tools]# ./runRemoteCmd.sh "jps" all

*******************cdh1***********************

26112 QuorumPeerMain

26261 Jps

26187 JournalNode

*******************cdh2***********************

26400 Jps

26256 QuorumPeerMain

26335 JournalNode

*******************cdh3***********************

26295 JournalNode

26361 Jps

26221 QuorumPeerMain

nn1 节点格式化 namenode

bin/hdfs namenode -format

nn1 节点格式化 zkfc

bin/hdfs zkfc -formatZK

nn1 节点启动 namenode

bin/hdfs namenode

nn2 节点同步 nn1 节点元数据信息

bin/hdfs namenode -bootstrapStandby

nn2 同步完 nn1 节点信息之后，Ctrl+c 关闭 nn1 节点 namenode 进程。

关闭所有节点 journalnode

./runRemoteCmd.sh "/root/app/hadoop/sbin/hadoop-daemon.sh stop journalnode" all

一键启动 hdfs

sbin/start-dfs.sh

[root@cdh1 hadoop]# sbin/start-dfs.sh

20/03/21 11:18:47 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

Starting namenodes on [cdh1 cdh2]

Starting journal nodes [cdh1 cdh2 cdh3]

Starting ZK Failover Controllers on NN hosts [cdh1 cdh2]

[root@cdh1 tools]# ./runRemoteCmd.sh "jps" all

*******************cdh1***********************

28960 Jps

26112 QuorumPeerMain

28871 DFSZKFailoverController

28314 NameNode

28603 JournalNode

28414 DataNode

*******************cdh2***********************

26256 QuorumPeerMain

27840 DFSZKFailoverController

27633 JournalNode

27905 Jps

27541 DataNode

27471 NameNode

*******************cdh3***********************

27222 JournalNode

27303 Jps

27129 DataNode

26221 QuorumPeerMain

hdfs 启动之后查看 namenode 状态

[root@cdh1 hadoop]# bin/hdfs haadmin -getServiceState nn1

active

[root@cdh1 hadoop]# bin/hdfs haadmin -getServiceState nn2

standby

一键关闭 hdfs

sbin/stop-dfs.sh

web查看hdfs

hosts文件

192.168.230.128 cdh1

192.168.230.129 cdh2

192.168.230.130 cdh3

http://cdh1:50070

active

http://cdh2:50070

standby

测试 hdfs

新建文件 wd.txt

[root@cdh1 app]# vi wd.txt

hadoop spark

Hdfs 文件系统创建 test 目录

bin/hdfs dfs -mkdir /test

[hadoop@master hadoop]$ bin/hdfs dfs -ls /

将 wd.txt 文件上传至/test 目录下

bin/hdfs dfs -put /root/app/wd.txt /test/

bin/hdfs dfs -ls /test

YARN

一键启动YARN

sbin/start-yarn.sh 一键启动 yarn 集群

启动备用节点 RM

sbin/yarn-daemon.sh start resourcemanager 启动备用节点

查看 RM 状态

[root@cdh1 hadoop]# bin/yarn rmadmin -getServiceState rm1

active

[root@cdh1 hadoop]# bin/yarn rmadmin -getServiceState rm2

Standby

web 界面查看 yarn

http://cdh1:8088/cluster

http://cdh2:8088/cluster跳转回http://cdh1:8088/cluster

测试运行 yarn

运行 Hadoop 自带 Wordcount 程序

bin/hadoop jar share/hadoop/mapreduce2/hadoop-mapreduce-examples-2.6.0-cdh5.10.0.jar wordcount /test/wd.txt /test/output

查看执行结果

[root@cdh1 hadoop]# bin/hdfs dfs -cat /test/output/*

hadoop 3

spark 3

总结

Hdfs集群搭建设计

服务进程	服务器分配	说明
zookeeper	cdh1,cdh2,cdh3	Zookeeper集群
namenode	cdh1,cdh2	namenode高可用
Datanade	cdh1,cdh2,cdh3	数据节点
Joulnalnode	cdh1,cdh2,cdh3	同步信息节点
Zfll	cdh1,cdh2	故障转移

Yarn集群设计

服务进程	服务器分配	说明
ResourceManger	cdh1,cdh2	cdh1,cdh2高可用
NodeManger	cdh1,cdh2,cdh3	namenode高可用
Zkfl	cdh1,cdh2	未独立出来的服务，包含在ResourceManger的进程中
Joulnalnode	cdh1,cdh2,cdh3	存储节点为zookeeper集群

运行后效果

lql_h

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
hadoop集群安装部署

HDFS解压 Hadooptar -xzvf hadoop-2.6.0-cdh5.10.0.tar.gz创建软连接ln -s hadoop-2.6.0-cdh5.10.0 hadoop修改配置文件路径/root/app/hadoop-2.6.0-cdh5.10.0/etc/hadoopcore-site.xml<configuration>...
复制链接

扫一扫