hadoop集群安装部署

HDFS

解压 Hadoop

tar -xzvf hadoop-2.6.0-cdh5.10.0.tar.gz

创建软连接

ln -s hadoop-2.6.0-cdh5.10.0 hadoop

修改配置文件

路径

/root/app/hadoop-2.6.0-cdh5.10.0/etc/hadoop

core-site.xml


<configuration>

        <property>

                <name>fs.defaultFS</name>

                <value>hdfs://mycluster</value>

        </property>

        <!--默认的HDFS路径-->

        <property>

                <name>hadoop.tmp.dir</name>

                <value>/root/data/tmp</value>

        </property>

        <!--hadoop的临时目录,如果需要配置多个目录,需要逗号隔开-->

        <property>

        <name>ha.zookeeper.quorum</name>

        <value>cdh1:2181,cdh2:2181,cdh3:2181</value>

        </property>

        <!--配置Zookeeper 管理HDFS-->

</configuration>


hdfs-site.xml

 


<configuration>

                <property>

                <name>dfs.replication</name>

                <value>3</value>

        </property>

                <!--数据块副本数为3-->

                <property>

                <name>dfs.permissions</name>

                <value>false</value>

        </property>

                <property>

                <name>dfs.permissions.enabled</name>

                <value>false</value>

        </property>

                <!--权限默认配置为false-->

                <property>

                <name>dfs.nameservices</name>

                <value>mycluster</value>

        </property>

                <!--命名空间,它的值与fs.defaultFS的值要对应,namenode高可用之后有两个namenode,mycluster是对外提供的统一入口-->

                <property>

                <name>dfs.ha.namenodes.mycluster</name>

                <value>nn1,nn2</value>

        </property>

                <!-- 指定 nameService 是 mycluster时的nameNode有哪些,这里的值也是逻辑名称,名字随便起,相互不重复即可-->

        <property>

                <name>dfs.namenode.rpc-address.mycluster.nn1</name>

                <value>cdh1:9000</value>

        </property>

        <property>

                <name>dfs.namenode.http-address.mycluster.nn1</name>

                <value>cdh1:50070</value>

        </property>

                <property>

                <name>dfs.namenode.rpc-address.mycluster.nn2</name>

                <value>cdh2:9000</value>

        </property>

        <property>

                <name>dfs.namenode.http-address.mycluster.nn2</name>

                <value>cdh2:50070</value>

        </property>

                <property>

                <name>dfs.ha.automatic-failover.enabled</name>

                <value>true</value>

        </property>

                <!--启动故障自动恢复-->

        <property>

                <name>dfs.namenode.shared.edits.dir</name>

                <value>qjournal://cdh1:8485;cdh2:8485;cdh3:8485/mycluster</value>

        </property>

                <!--指定NameNode的元数据在JournalNode上的存放位置-->

                <property>

                <name>dfs.client.failover.proxy.provider.mycluster</name>

                <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

        </property>

                <!--指定 mycluster 出故障时,哪个实现类负责执行故障切换-->

        <property>

                <name>dfs.journalnode.edits.dir</name>

                <value>/root/data/journaldata/jn</value>

        </property>

                <!-- 指定JournalNode在本地磁盘存放数据的位置 -->

                <property>

                <name>dfs.ha.fencing.methods</name>

                <value>shell(/bin/true)</value>

        </property>

                <!-- 配置隔离机制,shell通过ssh连接active namenode节点,杀掉进程-->

        <property>

                <name>dfs.ha.fencing.ssh.private-key-files</name>

                <value>/root/.ssh/id_rsa</value>

        </property>

                <!-- 为了实现SSH登录杀掉进程,还需要配置免密码登录的SSH密匙信息 -->

                <property>

                <name>dfs.ha.fencing.ssh.connect-timeout</name>

                <value>10000</value>

        </property>

        <property>

                <name>dfs.namenode.handler.count</name>

                <value>100</value>

        </property>

</configuration>

 

 

slaves


cdh1

cdh2

cdh3

 

hadoop-env.sh


export JAVA_HOME=/root/app/jdk

export HADOOP_HOME=/root/app/hadoop


yarn-site.xml

<configuration>

<property>

        <name>yarn.resourcemanager.connect.retry-interval.ms</name>

        <value>2000</value>

</property>

<property>

        <name>yarn.resourcemanager.ha.enabled</name>

        <value>true</value>

</property>

<!--打开高可用-->

<property>

        <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>

        <value>true</value>

</property>

<!--启动故障自动恢复-->

<property>

        <name>yarn.resourcemanager.ha.automatic-failover.embedded</name>

        <value>true</value>

</property>

<!--rm启动内置选举active-->

<property>

        <name>yarn.resourcemanager.cluster-id</name>

        <value>yarn-rm-cluster</value>

</property>

<!--给yarn cluster 取个名字yarn-rm-cluster-->

<property>

        <name>yarn.resourcemanager.ha.rm-ids</name>

        <value>rm1,rm2</value>

</property>

<!--ResourceManager高可用 rm1,rm2-->

<property>

        <name>yarn.resourcemanager.hostname.rm1</name>

        <value>cdh1</value>

</property>

<property>

        <name>yarn.resourcemanager.hostname.rm2</name>

        <value>cdh2</value>

</property>

<property>

        <name>yarn.resourcemanager.recovery.enabled</name>

        <value>true</value>

</property>

<!--启用resourcemanager 自动恢复-->

<property>

        <name>yarn.resourcemanager.zk.state-store.address</name>

        <value>cdh1:2181,cdh2:2181,cdh3:2181</value>

</property>

<!--状态存储地址-->

<property>

        <name>yarn.resourcemanager.zk-address</name>

        <value>cdh1:2181,cdh2:2181,cdh3:2181</value>

</property>

<!--配置Zookeeper地址-->

<property>

        <name>yarn.resourcemanager.address.rm1</name>

        <value>cdh1:8032</value>

</property>

<!--rm1端口号-->

<property>

        <name>yarn.resourcemanager.scheduler.address.rm1</name>

        <value>cdh1:8034</value>

</property>

<!-- rm1调度器的端口号-->

<property>

        <name>yarn.resourcemanager.webapp.address.rm1</name>

        <value>cdh1:8088</value>

</property>

<!-- rm1 webapp端口号-->

<property>

        <name>yarn.resourcemanager.address.rm2</name>

        <value>cdh2:8032</value>

</property>

<property>

        <name>yarn.resourcemanager.scheduler.address.rm2</name>

        <value>cdh2:8034</value>

</property>

<property>

        <name>yarn.resourcemanager.webapp.address.rm2</name>

        <value>cdh2:8088</value>

</property>

<property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

</property>

<property>

        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>

        <value>org.apache.hadoop.mapred.ShuffleHandler</value>

</property>

<!--执行MapReduce需要配置的shuffle过程-->

</configuration>

mapred-site


<configuration>

<property>

        <name>mapreduce.framework.name</name>

        <value>yarn</value>

</property>

<!--MapReduce以yarn模式运行-->

</configuration>

安装目录同步到其他节点

 ./deploy.sh /root/app/hadoop-2.6.0-cdh5.10.0 /root/app slave

各个节点分别创建软连接

ln -s hadoop-2.6.0-cdh5.10.0 hadoop

Hdfs格式化

./runRemoteCmd.sh "/root/app/zookeeper/bin/zkServer.sh start" all

./runRemoteCmd.sh "/root/app/hadoop/sbin/hadoop-daemon.sh start journalnode" all

 

[root@cdh1 tools]# ./runRemoteCmd.sh  "jps" all

*******************cdh1***********************

26112 QuorumPeerMain

26261 Jps

26187 JournalNode

*******************cdh2***********************

26400 Jps

26256 QuorumPeerMain

26335 JournalNode

*******************cdh3***********************

26295 JournalNode

26361 Jps

26221 QuorumPeerMain

nn1 节点格式化 namenode

bin/hdfs namenode -format

nn1 节点格式化 zkfc

bin/hdfs zkfc -formatZK

nn1 节点启动 namenode

bin/hdfs namenode

nn2 节点同步 nn1 节点元数据信息

bin/hdfs namenode -bootstrapStandby

nn2 同步完 nn1 节点信息之后,Ctrl+c 关闭 nn1 节点 namenode 进程。

关闭所有节点 journalnode

./runRemoteCmd.sh "/root/app/hadoop/sbin/hadoop-daemon.sh stop journalnode" all

一键启动 hdfs

sbin/start-dfs.sh

[root@cdh1 hadoop]# sbin/start-dfs.sh

20/03/21 11:18:47 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

Starting namenodes on [cdh1 cdh2]

Starting journal nodes [cdh1 cdh2 cdh3]

Starting ZK Failover Controllers on NN hosts [cdh1 cdh2]

 

[root@cdh1 tools]# ./runRemoteCmd.sh  "jps" all

*******************cdh1***********************

28960 Jps

26112 QuorumPeerMain

28871 DFSZKFailoverController

28314 NameNode

28603 JournalNode

28414 DataNode

*******************cdh2***********************

26256 QuorumPeerMain

27840 DFSZKFailoverController

27633 JournalNode

27905 Jps

27541 DataNode

27471 NameNode

*******************cdh3***********************

27222 JournalNode

27303 Jps

27129 DataNode

26221 QuorumPeerMain

hdfs 启动之后查看 namenode 状态

[root@cdh1 hadoop]# bin/hdfs haadmin -getServiceState nn1

active

[root@cdh1 hadoop]# bin/hdfs haadmin -getServiceState nn2

standby

一键关闭 hdfs

sbin/stop-dfs.sh

web查看hdfs

hosts文件

192.168.230.128 cdh1

192.168.230.129 cdh2

192.168.230.130 cdh3

 

http://cdh1:50070

active

 

http://cdh2:50070

standby

测试 hdfs

新建文件 wd.txt

[root@cdh1 app]# vi wd.txt

hadoop spark

hadoop spark

hadoop spark

Hdfs 文件系统创建 test 目录

bin/hdfs dfs -mkdir /test

[hadoop@master hadoop]$ bin/hdfs dfs -ls /

将 wd.txt 文件上传至/test 目录下

bin/hdfs dfs -put /root/app/wd.txt /test/

bin/hdfs dfs -ls /test

YARN

一键启动YARN

sbin/start-yarn.sh 一键启动 yarn 集群

启动备用节点 RM

sbin/yarn-daemon.sh start resourcemanager 启动备用节点

查看 RM 状态

[root@cdh1 hadoop]# bin/yarn rmadmin -getServiceState rm1

active

[root@cdh1 hadoop]# bin/yarn rmadmin -getServiceState rm2

Standby

web 界面查看 yarn

http://cdh1:8088/cluster

http://cdh2:8088/cluster跳转回http://cdh1:8088/cluster

测试运行 yarn

运行 Hadoop 自带 Wordcount 程序

bin/hadoop jar share/hadoop/mapreduce2/hadoop-mapreduce-examples-2.6.0-cdh5.10.0.jar wordcount /test/wd.txt /test/output

查看执行结果

[root@cdh1 hadoop]# bin/hdfs dfs -cat /test/output/*

hadoop  3

spark   3

总结

Hdfs集群搭建设计

服务进程

服务器分配

说明

zookeeper

cdh1,cdh2,cdh3

Zookeeper集群

namenode

cdh1,cdh2

namenode高可用

Datanade

cdh1,cdh2,cdh3

数据节点

Joulnalnode

cdh1,cdh2,cdh3

同步信息节点

Zfll

cdh1,cdh2

故障转移

 

Yarn集群设计

服务进程

服务器分配

说明

ResourceManger

cdh1,cdh2

cdh1,cdh2高可用

NodeManger

cdh1,cdh2,cdh3

namenode高可用

Zkfl

cdh1,cdh2

未独立出来的服务,包含在ResourceManger的进程中

Joulnalnode

cdh1,cdh2,cdh3

存储节点为zookeeper集群

运行后效果

 

发布了162 篇原创文章 · 获赞 7 · 访问量 1万+
展开阅读全文

没有更多推荐了,返回首页

©️2019 CSDN 皮肤主题: 大白 设计师: CSDN官方博客

分享到微信朋友圈

×

扫一扫,手机浏览