部署kafka集群

环境

centos7.5
[root@node-2 config]# java -version
openjdk version "1.8.0_262"

cat >> /etc/hosts << EOF
192.168.2.128 node-1
192.168.2.129 node-2
192.168.2.130 node-3
EOF

yum install java-1.8.0-openjdk -y

mkdir -p /soft/src
cd /soft/src

下载kafka
wget https://mirrors.cnnic.cn/apache/kafka/0.11.0.2/kafka_2.12-2.8.0.tgz （网址不一定正确，版本是这个）


tar xf kafka_2.12-2.8.0.tgz
mv kafka_2.12-2.8.0 kafka
scp -r kafka root@node-2:/soft/src/
scp -r kafka root@node-3:/soft/src/

配置zookeeper集群（kafka将zookeeper做数据库，这边直接用kafka自带的）

cd kafka/config/
[root@node-2 config]# egrep -v '^#|^$' zookeeper.properties
dataDir=/soft/src/kafka/zookeeper
clientPort=2181
maxClientCnxns=0
initLimit=10
syncLimit=5
server.1=node-1:2888:3888
server.2=node-2:2889:3889
server.3=node-3:2890:3890

scp /soft/src/kafka/config/zookeeper.properties root@node-2:/soft/src/kafka/config
scp /soft/src/kafka/config/zookeeper.properties root@node-3:/soft/src/kafka/config

ssh root@node-2 'mkdir /soft/src/kafka/zookeeper'
ssh root@node-2 'echo 2 > /soft/src/kafka/zookeeper/myid'

ssh root@node-3 'mkdir /soft/src/kafka/zookeeper'
ssh root@node-3 'echo 3 > /soft/src/kafka/zookeeper/myid'

启动zookeeper
/soft/src/kafka/bin/zookeeper-server-start.sh -daemon /soft/src/kafka/config/zookeeper.properties'
验证是否启动，有时候没启动也不报错
ps -ef|grep zookeeper.properties

配置kafka集群

[root@localhost config]$egrep -v '^#|^$' server.properties	#### 注意，broker.id后面只能写数字，写注释一定会报错
broker.id=1     #三个节点不能一样
listeners=PLAINTEXT://:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/usr/local/kafka/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.106.7:2181,192.168.106.8:2181,192.168.106.9:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0

scp /soft/src/kafka/config/server.properties root@node-2:/soft/src/
scp /soft/src/kafka/config/server.properties root@node-3:/soft/src/

启动kafka
/soft/src/kafka/bin/kafka-server-start.sh -daemon /soft/src/kafka/config/server.properties
验证
pe -ef|grep server.properties

创建topic
/soft/src/kafka/bin/kafka-topics.sh --create --zookeeper 192.168.2.128:2181,192.168.2.129:2181,192.168.2.130:2181 --replication-factor 3 --partitions 1 --topic test-topic


查看topic
/soft/src/kafka/bin/kafka-topics.sh --list --zookeeper 192.168.2.128:2181,192.168.2.129:2181,192.168.2.130:2181


生产消息

/soft/src/kafka/bin/kafka-console-producer.sh --broker-list 192.168.2.128:9092,192.168.2.129:9092,192.168.2.130:9092 --topic test-topic

消费消息
/soft/src/kafka/bin/kafka-console-consumer.sh --bootstrap-server 192.168.2.128:9092,192.168.2.129:9092,192.168.2.130:9092 --topic test-topic --from-beginning