CentOS 7 下 Zookeeper + Kafka 集群安装配置

服务器信息

主机名	IP	CPU	内存	磁盘	部署
node1	10.10.200.211	2核	4GB	50GB	zookeeper kafka
node2	10.10.200.212	2核	4GB	50GB	zookeeper kafka
node3	10.10.200.213	2核	4GB	50GB	zookeeper kafka

Zookeeper 安装配置

安装包准备

[root@node1 soft]# ll | grep zookeeper
-rw-r--r--. 1 root root  12436328 Aug 11 14:48 apache-zookeeper-3.6.1-bin.tar.gz

解压安装包

[root@node1 ~]# tar -zxf apache-zookeeper-3.6.1-bin.tar.gz && mv apache-zookeeper-3.6.1-bin /usr/local/zookeeper

配置

复制配置文件

[root@node1 ~]# cd /usr/local/zookeeper/conf && cp zoo_sample.cfg zoo.cfg

配置 zoo.cfg

[root@node1 ~]# cat zoo.cfg | grep -v '^#' | grep -v '^$'
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/local/zookeeper/data
dataLogDir=/usr/local/zookeeper/logs
clientPort=2181
extendedTypesEnabled=true
emulate353TTLNodes=true
admin.enableServer=false
4lw.commands.whitelist=*
server.1=10.10.200.211:2888:3888
server.2=10.10.200.212:2888:3888
server.3=10.10.200.213:2888:3888

对应目录创建

[root@node1 ~]# mkdir /usr/local/zookeeper/{data,logs}
[root@node1 ~]# echo "1" > /usr/local/zookeeper/data/myid

Zookeeper 环境变量配置

[root@node1 ~]# tail -n 3 /etc/profile
# Set Zookeeker Env
export ZOOKEEPER_HOME=/usr/local/zookeeper
export PATH=$PATH:$ZOOKEEPER_HOME/bin

刷新环境变量

[root@node1 ~]# source /etc/profile

启动

[root@node1 ~]# zkServer.sh start

连接测试

zkCli 连接测试

[root@node1 ~]# zkCli.sh -server node1:2181

zkServer 状态查看

[root@node1 ~]# zkServer.sh status

Kafka 安装配置

安装包准备

[root@node1 soft]# ll | grep kafka
-rw-r--r--. 1 root root  65671917 Aug 26 15:35 kafka_2.12-2.6.0.tgz

解压安装包

[root@node1 ~]# tar -zxf kafka_2.12-2.6.0.tgz && mv kafka_2.12-2.6.0 /usr/local/kafka

配置

Kafka 配置文件配置

[root@node1 config]# cat /usr/local/kafka/config/server.properties | grep -v ^$ | grep -v ^#
broker.id=1
listeners=PLAINTEXT://node1:9092
host.name=node1
port=9092
advertised.listeners=PLAINTEXT://node1:9092
advertised.host.name=node1
advertised.port=9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/usr/local/kafka/logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=node1:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

对应目录创建

[root@node1 ~]# mkdir /usr/local/kafka/logs

环境变量配置

[root@node1 ~]# tail -n 3 /etc/profile
# Set Kafka Env
export KAFKA_HOME=/usr/local/kafka
export PATH=$PATH:$KAFKA_HOME/bin

刷新环境变量

[root@node1 ~]# source /etc/profile

启动

[root@node1 ~]# /usr/local/kafka/bin/kafka-server-start.sh -daemon /usr/local/kafka/config/server.properties

Kafka 可用性验证

新建 Topic

[root@node1 ~]# kafka-topics.sh --create --zookeeper node1:2181 --replication-factor 3 --partitions 3 --topic test
Created topic test.

查看 Topic

[root@node1 ~]# kafka-topics.sh --list --zookeeper node1:2181
test

[root@node1 ~]# kafka-topics.sh --list --zookeeper node2:2181
test

[root@node1 ~]# kafka-topics.sh --list --zookeeper node3:2181
test

[root@node1 ~]# kafka-topics.sh --describe --zookeeper node1:2181 --topic test
Topic: test	PartitionCount: 3	ReplicationFactor: 3	Configs: 
	Topic: test	Partition: 0	Leader: 2	Replicas: 2,3,1	Isr: 2,3,1
	Topic: test	Partition: 1	Leader: 3	Replicas: 3,1,2	Isr: 3,1,2
	Topic: test	Partition: 2	Leader: 1	Replicas: 1,2,3	Isr: 1,2,3

启动消息生产者

[root@node1 ~]# kafka-console-producer.sh --broker-list node1:9092 --topic test

启动消息消费者

[root@node2 ~]# kafka-console-consumer.sh --bootstrap-server node2:9092 --from-beginning --topic test