kafka_2.13-2.8.0

qq_26264237

已于 2023-08-03 17:20:47 修改

阅读量853

点赞数

文章标签： kafka java 分布式

于 2023-06-26 11:39:35 首次发布

本文链接：https://blog.csdn.net/qq_26264237/article/details/130381421

版权

kafka_2.13-2.8.0

下载kafka
启动zookeeper
启动kafka
- server.properties 单机版配置
验证zk及kafka是否启动成功
启动停止kafka
- 启动kafka脚本
- 关闭kafka脚本
创建主题 / 删除主题 / 查看主题
主题消息生产及消费
- 消费分组查看

下载kafka

https://kafka.apache.org/downloads
https://archive.apache.org/dist/kafka/2.8.0/kafka_2.13-2.8.0.tgz

启动zookeeper

./bin/zookeeper-server-start.sh -daemon config/zookeeper.properties

zookeeper.properties

#存放zk数据信息
dataDir=/opt/kafka/kafka_2.13-2.8.0/data/zookeeper
clientPort=2181
maxClientCnxns=0
admin.enableServer=false

zookeeper是否启动成功

lsof -i :2181

配置文件指定了端口2181，通过查看端口检查zookeeper是否启动成功

[root@ZHCS-YWZTJX-AP02 kafka_2.13-2.8.0]# lsof -i :2181
COMMAND   PID USER   FD   TYPE    DEVICE SIZE/OFF NODE NAME
java    32823 root  125u  IPv6 560416769      0t0  TCP *:eforward (LISTEN)

ps -aux | grep zookeeper

[root@ZHCS-YWZTJX-AP02 kafka_2.13-2.8.0]# ps -aux | grep zookeeper

启动kafka

./bin/kafka-server-start.sh -daemon config/server.properties

server.properties 单机版配置

#broker的全局唯一编号，不能重复
broker.id=0

#监听地址
listeners=PLAINTEXT://:9092

#公开监听地址
advertised.listeners=PLAINTEXT://43.139.219.171:9092

#日志文件目录
log.dirs=/usr/local/kafka/kf_log

#ZK地址
zookeeper.connect=localhost:2181
zookeeper.connection.timeout.ms=18000

offsets.topic.replication.factor=1
config.storage.replication.factor=1
offset.storage.replication.factor=1
status.storage.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1

listeners和advertised.listeners的地址不能相同，listeners填写内部地址advertised.listeners填写外部地址
需要防火墙放开9092端口

listeners和advertised.listeners的地址相同会报错如下：

org.apache.kafka.common.KafkaException: Socket server failed to bind to 43.139.219.171:9092: Cannot assign requested address.
        at kafka.network.Acceptor.openServerSocket(SocketServer.scala:667)
        at kafka.network.Acceptor.<init>(SocketServer.scala:560)
        at kafka.network.SocketServer.createAcceptor(SocketServer.scala:288)
        at kafka.network.SocketServer.$anonfun$createDataPlaneAcceptorsAndProcessors$1(SocketServer.scala:261)
        at kafka.network.SocketServer.$anonfun$createDataPlaneAcceptorsAndProcessors$1$adapted(SocketServer.scala:259)
        at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:563)
        at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:561)
        at scala.collection.AbstractIterable.foreach(Iterable.scala:919)
        at kafka.network.SocketServer.createDataPlaneAcceptorsAndProcessors(SocketServer.scala:259)
        at kafka.network.SocketServer.startup(SocketServer.scala:131)
        at kafka.server.KafkaServer.startup(KafkaServer.scala:285)
        at kafka.Kafka$.main(Kafka.scala:109)
        at kafka.Kafka.main(Kafka.scala)
Caused by: java.net.BindException: Cannot assign requested address
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:461)
        at sun.nio.ch.Net.bind(Net.java:453)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:222)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:85)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:78)
        at kafka.network.Acceptor.openServerSocket(SocketServer.scala:663)
        ... 12 more

如果是本机：zookeeper.connect=localhost:2181
对外提供端口则填写，对外ip地址，注意防火墙放开2181端口

lsof -i :9092
配置文件指定了端口9092，通过查看端口检查kafka是否启动成功

验证zk及kafka是否启动成功

[root@DEV-YWZTV1-SE05 kafka_2.13-2.8.0]# jps
96867 Kafka
96482 QuorumPeerMain
96904 Jps

启动停止kafka

# 启动kafka
[root@sit-crmcp-kafka01 /]# ./bin/kafka-server-start.sh config/server.properties
# 守护进程启动kafka
[root@sit-crmcp-kafka01 /]# ./bin/kafka-server-start.sh -daemon config/server.properties
# 守护进程启动kafka 或者使用nohup
[root@sit-crmcp-kafka01 /]# nohup ./bin/kafka-server-start.sh config/server.properties &
# 停止kafka
[root@sit-crmcp-kafka01 /]# ./bin/kafka-server-stop.sh

为方便启动停止kafka，创建脚本，非必须

启动kafka脚本

#!/bin/bash
cd /mpjava/kafka_2.13-2.8.0
nohup ./bin/kafka-server-start.sh config/server.properties &
echo "启动kafka"

关闭kafka脚本

#!/bin/bash
cd /mpjava/kafka_2.13-2.8.0/
./bin/kafka-server-stop.sh
echo "关闭kafka"

创建主题 / 删除主题 / 查看主题

# 列出所有主题
[root@sit-crmcp-kafka01 /]# ./bin/kafka-topics.sh --zookeeper 43.139.219.171:2181 --list
# 主题查看
[root@sit-crmcp-kafka01 /]# ./bin/kafka-topics.sh --zookeeper 43.139.219.171:2181 --describe  --topic mytopic
# 创建主题
[root@sit-crmcp-kafka01 /]# ./bin/kafka-topics.sh --zookeeper 43.139.219.171 --create --topic mytopic --partitions 1 --replication-factor 1
# 主题删除
[root@sit-crmcp-kafka01 /]# ./bin/kafka-topics.sh --zookeeper 43.139.219.171:2181 --delete  --topic mytopic

主题消息生产及消费

# 生产主题消息
[root@sit-crmcp-kafka01 /]# ./bin/kafka-console-producer.sh --broker-list 43.139.219.171:9092 --topic mytopic
# 消费主题消息
[root@sit-crmcp-kafka01 /]# ./bin/kafka-console-consumer.sh --bootstrap-server 43.139.219.171:9092 --topic mytopic --from-beginning

消费分组查看

# 查看所有分组
[root@ZHCS-YWZTJX-AP03 bin]# ./kafka-consumer-groups.sh --bootstrap-server 172.26.240.155:9092 --list
ly-mp-csc-clue-service_fe6dd877-623e-46f9-b7e0-8c47c2a53fba
ly-mp-csc-clue-service_b4293fe9-e120-4040-b195-bdd09da3a1eb
ly-mp-csc-clue-service_2b8a8115-8f06-4a5c-aba1-52a9c7da3436
bucn.ms
ly-mp-csc-clue-service_cf6a1a34-e243-4aaa-a209-ac31bd37ae3a
ly-mp-csc-clue-service_a75706e4-ba9e-402d-bcd9-cee1f570726b

#查看具体分组内容
[root@ZHCS-YWZTJX-AP03 bin]# ./kafka-consumer-groups.sh --bootstrap-server 172.26.240.155:9092 --group bucn.ms --describe

GROUP           TOPIC                       PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG             CONSUMER-ID                                             HOST            CLIENT-ID
bucn.ms         toc-CouponIssueMq           2          -               0               -               consumer-bucn.ms-1-0d9ed52a-a790-4e89-882f-297476c817c6 /172.26.240.155 consumer-bucn.ms-1
bucn.ms         toc-CouponIssueMq           4          -               0               -               consumer-bucn.ms-1-0d9ed52a-a790-4e89-882f-297476c817c6 /172.26.240.155 consumer-bucn.ms-1
bucn.ms         toc-CouponIssueMq           1          -               0               -               consumer-bucn.ms-1-0d9ed52a-a790-4e89-882f-297476c817c6 /172.26.240.155 consumer-bucn.ms-1
bucn.ms         toc-CouponIssueMq           0          -               0               -               consumer-bucn.ms-1-0d9ed52a-a790-4e89-882f-297476c817c6 /172.26.240.155 consumer-bucn.ms-1
bucn.ms         toc-CouponIssueMq           3          -               0               -               consumer-bucn.ms-1-0d9ed52a-a790-4e89-882f-297476c817c6 /172.26.240.155 consumer-bucn.ms-1
bucn.ms         toc-CouponCUserReceiveMq    3          50              50              0               consumer-bucn.ms-4-3aee0d57-f38c-4c8f-b856-8479eb2d89f7 /172.26.240.155 consumer-bucn.ms-4
bucn.ms         toc-CouponCUserReceiveMq    2          36              36              0               consumer-bucn.ms-4-3aee0d57-f38c-4c8f-b856-8479eb2d89f7 /172.26.240.155 consumer-bucn.ms-4
bucn.ms         toc-CouponCUserReceiveMq    1          50              50              0               consumer-bucn.ms-4-3aee0d57-f38c-4c8f-b856-8479eb2d89f7 /172.26.240.155 consumer-bucn.ms-4
bucn