本文是我学习Hadoop搭建过程中的各种笔记,内容来自于各种公开的教程,起点非常低,从Linux基础开始,直至在PC上搭建Hadoop成功,是真正的从零开始。
感谢过程中帮助我的各位认识的和不认识的老师。
48、Zookeeper集群的搭建:
Zookeeper保证分布式服务更加高可靠,一般是3台、5台、7台;单机版,集群
下载:
[root@hadoop01~]# cd /home/
[root@hadoop01 home]# rz
[root@hadoop01 home]# tar –zxvf /home/ zookeeper-3.4.12.tar.gz –C /usr/local/
[root@hadoop01 home]# cd /usr/local/
[root@hadoop01 local]# ll
total 56
………………
drwxr-xr-x. 10 1000 1000 4096 Mar 27 12:36 zookeeper-3.4.12
[root@hadoop01 local]# ll ./zookeeper-3.4.12/
total 1624
drwxr-xr-x. 2 1000 1000 4096 Mar 27 12:32 bin
-rw-rw-r--. 1 1000 1000 87945 Mar 27 12:32 build.xml
drwxr-xr-x. 2 1000 1000 4096 Mar 27 12:32 conf
drwxr-xr-x. 10 1000 1000 4096 Mar 27 12:32 contrib
drwxr-xr-x. 2 1000 1000 4096 Mar 27 12:37 dist-maven
drwxr-xr-x. 6 1000 1000 4096 Mar 27 12:32 docs
-rw-rw-r--. 1 1000 1000 1709 Mar 27 12:32 ivysettings.xml
-rw-rw-r--. 1 1000 1000 8197 Mar 27 12:32 ivy.xml
drwxr-xr-x. 4 1000 1000 4096 Mar 27 12:32 lib
-rw-rw-r--. 1 1000 1000 11938 Mar 27 12:32 LICENSE.txt
-rw-rw-r--. 1 1000 1000 3132 Mar 27 12:32 NOTICE.txt
-rw-rw-r--. 1 1000 1000 1585 Mar 27 12:32 README.md
-rw-rw-r--. 1 1000 1000 1770 Mar 27 12:32 README_packaging.txt
drwxr-xr-x. 5 1000 1000 4096 Mar 27 12:32 recipes
drwxr-xr-x. 8 1000 1000 4096 Mar 27 12:32 src
-rw-rw-r--. 1 1000 1000 1483366 Mar 27 12:32 zookeeper-3.4.12.jar
-rw-rw-r--. 1 1000 1000 819 Mar 27 12:36 zookeeper-3.4.12.jar.asc
-rw-rw-r--. 1 1000 1000 33 Mar 27 12:32 zookeeper-3.4.12.jar.md5
-rw-rw-r--. 1 1000 1000 41 Mar 27 12:32 zookeeper-3.4.12.jar.sha1
docs很杂,一会要移动zookeeper-3.4.12.docs会麻烦,所以删除:
[root@hadoop01 local]# rm -rf ./zookeeper-3.4.12/docs/
[root@hadoop01 local]# cd ./zookeeper-3.4.12/
[root@hadoop01 zookeeper-3.4.12]# ll
total 1620
drwxr-xr-x. 2 1000 1000 4096 Mar 27 12:32 bin
-rw-rw-r--. 1 1000 1000 87945 Mar 27 12:32 build.xml
drwxr-xr-x. 2 1000 1000 4096 Mar 27 12:32 conf
drwxr-xr-x. 10 1000 1000 4096 Mar 27 12:32 contrib
drwxr-xr-x. 2 1000 1000 4096 Mar 27 12:37 dist-maven
-rw-rw-r--. 1 1000 1000 1709 Mar 27 12:32 ivysettings.xml
-rw-rw-r--. 1 1000 1000 8197 Mar 27 12:32 ivy.xml
drwxr-xr-x. 4 1000 1000 4096 Mar 27 12:32 lib
-rw-rw-r--. 1 1000 1000 11938 Mar 27 12:32 LICENSE.txt
-rw-rw-r--. 1 1000 1000 3132 Mar 27 12:32 NOTICE.txt
-rw-rw-r--. 1 1000 1000 1585 Mar 27 12:32 README.md
-rw-rw-r--. 1 1000 1000 1770 Mar 27 12:32 README_packaging.txt
drwxr-xr-x. 5 1000 1000 4096 Mar 27 12:32 recipes
drwxr-xr-x. 8 1000 1000 4096 Mar 27 12:32 src
-rw-rw-r--. 1 1000 1000 1483366 Mar 27 12:32 zookeeper-3.4.12.jar
-rw-rw-r--. 1 1000 1000 819 Mar 27 12:36 zookeeper-3.4.12.jar.asc
-rw-rw-r--. 1 1000 1000 33 Mar 27 12:32 zookeeper-3.4.12.jar.md5
-rw-rw-r--. 1 1000 1000 41 Mar 27 12:32 zookeeper-3.4.12.jar.sha1
配置环境变量:
[root@hadoop01 zookeeper-3.4.12]# vi /etc/profile
my settings
JAVA_HOME=/usr/local/jdk1.8.0_144/
HADOOP_HOME=/usr/local/hadoop-2.7.1/
ZK_HOME=/usr/local/zookeeper-3.4.12/
PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin
[root@hadoop01 zookeeper-3.4.12]# source /etc/profile
[root@hadoop01 zookeeper-3.4.12]# ll ./bin/
total 36
-rwxr-xr-x. 1 1000 1000 232 Mar 27 12:32 README.txt
-rwxr-xr-x. 1 1000 1000 1937 Mar 27 12:32 zkCleanup.sh
-rwxr-xr-x. 1 1000 1000 1056 Mar 27 12:32 zkCli.cmd
-rwxr-xr-x. 1 1000 1000 1534 Mar 27 12:32 zkCli.sh
-rwxr-xr-x. 1 1000 1000 1759 Mar 27 12:32 zkEnv.cmd
-rwxr-xr-x. 1 1000 1000 2696 Mar 27 12:32 zkEnv.sh
-rwxr-xr-x. 1 1000 1000 1089 Mar 27 12:32 zkServer.cmd
-rwxr-xr-x. 1 1000 1000 6773 Mar 27 12:32 zkServer.sh
[root@hadoop01 zookeeper-3.4.12]# zkServer.sh (可以Tab出来,没问题)
[root@hadoop01 zookeeper-3.4.12]# ll ./conf/
total 12
-rw-rw-r--. 1 1000 1000 535 Mar 27 12:32 configuration.xsl
-rw-rw-r--. 1 1000 1000 2161 Mar 27 12:32 log4j.properties
-rw-rw-r--. 1 1000 1000 922 Mar 27 12:32 zoo_sample.cfg
配zoo_sample.cfg,这个是一个模板文件默认是.cfg,修改,为zoo.cfg:
[root@hadoop01 zookeeper-3.4.12]# mv ./conf/zoo_sample.cfg ./conf/zoo.cfg
[root@hadoop01 zookeeper-3.4.12]# vi ./conf/zoo.cfg
# The number of milliseconds of each tick (ZK每一个单元的单元格,ZK中用到的单元都基于这个单元的)
tickTime=2000 (2000毫秒)
# The number of ticks that the initial
# synchronization phase can take
initLimit=10 (10改成5)
# The number of ticks that can pass between (删除)
# sending a request and getting an acknowledgement (启动时小弟像老大汇报数据,发送一个请求,接受到时间的请求;时间发送消息和接送到消息的时间,超时就证明超时了,删除)
syncLimit=5 (5改成2)
# the directory where the snapshot is stored. (快件存储目录)
# do not use /tmp for storage, /tmp here is just (不要用这个存储,仅仅是一个例子,/tmp是不受我们控制的,删除)
# example sakes. (删除)
dataDir=/tmp/zookeeper (修改为:dataDir=/home/zkdata/)
# the port at which the clients will connect (端口是客户端连接的端口,删除)
clientPort=2181
# the maximum number of client connections. (删除)
# increase this if you need to handle more clients (删除)
#maxClientCnxns=60 (删除)
# (删除)
"./conf/zoo.cfg" 28L, 922C (删除)
把官网上的拿出来:
网址:https://zookeeper.apache.org/doc/current/zookeeperAdmin.html
server.1=zoo1:2888:3888
server.2=zoo2:2888:3888
server.3=zoo3:2888:3888
三台集群,三个服务,1/2/3只要不重复就行,zoo1,zoo2,zoo3为主机名,修改为hadoop01,02,03;2888是指小弟和老大的通信端口,3888是指老大挂掉后要在小弟中选举一个出来,选举端口;用伪分布式要变
最终配置结果如下:
tickTime=2000
initLimit=5
syncLimit=2
dataDir=/home/zkdata/
clientPort=2181
server.1=hadoop01:2888:3888
server.2=hadoop02:2888:3888
server.3=hadoop03:2888:3888
2888:小弟和老大通信端口;3888:选举端口。(用伪分布式得变)
将目录zookeeper-3.4.12远程cp到hadoop02和hadoop03上:
[root@hadoop01 zookeeper-3.4.12]# scp -r ../zookeeper-3.4.12/ hadoop02:/usr/local/
[root@hadoop01 zookeeper-3.4.12]# scp -r ../zookeeper-3.4.12/ hadoop03:/usr/local/
在hadoop02上配置环境变量:
[root@hadoop02 ~]# vi /etc/profile
JAVA_HOME=/usr/local/jdk1.8.0_144/
HADOOP_HOME=/usr/local/hadoop-2.7.1/
ZK_HOME=/usr/local/zookeeper-3.4.12/
PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:
[root@hadoop02 ~]# source /etc/profile
[root@hadoop02 ~]# zk
zkCleanup.sh zkCli.cmd zkCli.sh zkEnv.cmd zkEnv.sh zkServer.cmd zkServer.sh
[root@hadoop02 ~]# zkServer. (Tab能出来,环境变量配置好)
在hadoop03上配置环境变量:
[root@hadoop03 ~]# vi /etc/profile
[root@hadoop03 ~]# source /etc/profile
[root@hadoop03 ~]# zkServer. (Tab能出来,环境变量配置好)
现在还不行,要在每台服务器上创建这个目录dataDir=/home/zkdata/:
在每一台上面创建下面这个目录:
[root@hadoop01 zookeeper-3.4.12]# mkdir /home/zkdata/
[root@hadoop02 zookeeper-3.4.12]# mkdir /home/zkdata/
[root@hadoop03 zookeeper-3.4.12]# mkdir /home/zkdata/
在hadoop01:
[root@hadoop01 zookeeper-3.4.12]# vi /home/zkdata/myid
输入:1
原因:
server.1=hadoop01:2888:3888
server.2=hadoop02:2888:3888
server.3=hadoop03:2888:3888
在hadoop02:
[root@hadoop02 ~]# vi /home/zkdata/myid
输入:2
在hadoop03:
[root@hadoop03 ~]# vi /home/zkdata/myid
输入:3
启动:
[root@hadoop01 zookeeper-3.4.12]# vi ./conf/zoo.cfg
【东西就在./conf/zoo.cfg下面,所以zkServer.sh start后面不要加东西,如果不在,要写成:zkServer.sh start ./conf/zk.cfg(比如:zk.cfg)】
在hadoop01上启动zkServer:
[root@hadoop01 zookeeper-3.4.12]# zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper-3.4.12/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
现在zkServer还不可以用,奇数台机子,要启动一半才可以用,要在hadoop02上启动。
在hadoop02上启动zkServer:
[root@hadoop02 ~]# zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper-3.4.12/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
此时,在hadoop01上查看,就多了一个QuorumPeerMain,这个就是ZK
[root@hadoop01 zookeeper-3.4.12]# jps
12966 QuorumPeerMain
用另一种方式在hadoop01上查看:
[root@hadoop01 zookeeper-3.4.12]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper-3.4.12/bin/../conf/zoo.cfg
Error contacting service. It is probably not running.【联系服务时出错。 它可能没有运行 】
(出现问题,解决:http://www.aboutyun.com/thread-18791-1-1.html )
正确的应该是:
在hadoop01上:
[root@hadoop01 zookeeper-3.4.12]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper-3.4.12/bin/../conf/zoo.cfg
Mode:follower 【小弟的意思】
在hadoop02上启动:
[root@hadoop02 ~]# zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper-3.4.12/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@hadoop02 zookeeper-3.4.12]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper-3.4.12/bin/../conf/zoo.cfg
Mode:leader 【老大的意思】
在hadoop03上启动:
[root@hadoop03 ~]# zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper-3.4.12/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@hadoop02 zookeeper-3.4.12]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper-3.4.12/bin/../conf/zoo.cfg
Mode:follower 【小弟的意思】
zk集群此时装好,用客户端连接他:sh后面不加任何东西,任务是连接当前的这台服务,在hadoop01上连接,连接的是follower
[root@hadoop01 zookeeper-3.4.12]# zkCli.sh
出来的应该是(ls根目录,就能看到zookeeper节点):
[zk: localhost:2181 (CONNECTED) 0] ls /
[zookeeper]
退出去:
[zk: localhost:2181 (CONNECTED) 0] quit
Zookeeper的搭建↑