rocky8.6与ubuntu2004一键安装kafka集群

最新推荐文章于 2024-07-06 21:54:55 发布

lh1107

最新推荐文章于 2024-07-06 21:54:55 发布

阅读量426

点赞数 7

文章标签： kafka 分布式

本文链接：https://blog.csdn.net/lh1107/article/details/136150084

版权

kafka

#注意:生产中zookeeper和kafka一般是分开独立部署的,kafka安装前需要安装java环境

kafka配置文件说明

#配置文件 ./conf/server.properties内容说明

############################# Server Basics###############################
# broker的id，值为整数，且必须唯一，在一个集群中不能重复
broker.id=1

############################# Socket ServerSettings ######################
# kafka监听端口，默认9092
listeners=PLAINTEXT://10.0.0.101:9092
# 处理网络请求的线程数量，默认为3个
num.network.threads=3
# 执行磁盘IO操作的线程数量，默认为8个
num.io.threads=8
# socket服务发送数据的缓冲区大小，默认100KB
socket.send.buffer.bytes=102400
# socket服务接受数据的缓冲区大小，默认100KB
socket.receive.buffer.bytes=102400
# socket服务所能接受的一个请求的最大大小，默认为100M
socket.request.max.bytes=104857600

############################# Log Basics###################################
# kafka存储消息数据的目录
log.dirs=../data
# 每个topic默认的partition
num.partitions=1
# 设置副本数量为3,当Leader的Replication故障，会进行故障自动转移。
default.replication.factor=3
# 在启动时恢复数据和关闭时刷新数据时每个数据目录的线程数量
num.recovery.threads.per.data.dir=1

############################# Log FlushPolicy #############################
# 消息刷新到磁盘中的消息条数阈值
log.flush.interval.messages=10000
# 消息刷新到磁盘中的最大时间间隔,1s
log.flush.interval.ms=1000

############################# Log RetentionPolicy #########################
# 日志保留小时数，超时会自动删除，默认为7天
log.retention.hours=168
# 日志保留大小，超出大小会自动删除，默认为1G
#log.retention.bytes=1073741824
# 日志分片策略，单个日志文件的大小最大为1G，超出后则创建一个新的日志文件
log.segment.bytes=1073741824
# 每隔多长时间检测数据是否达到删除条件,300s
log.retention.check.interval.ms=300000

############################# Zookeeper ####################################
# Zookeeper连接信息，如果是zookeeper集群，则以逗号隔开
zookeeper.connect=10.0.0.101:2181,10.0.0.102:2181,10.0.0.103:2181
# 连接zookeeper的超时时间,6s
zookeeper.connection.timeout.ms=6000

kafka集群脚本

修改脚本里对应的主机ip
在这里插入图片描述

vim install_kafka.sh

#!/bin/bash

KAFKA_VERSION=3.6.0
SCALA_VERSION=2.13
KAFKA_URL="https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz"

ZK_VERSOIN=3.8.3
ZK_URL="https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/stable/apache-zookeeper-${ZK_VERSOIN}-bin.tar.gz"

ZK_INSTALL_DIR=/usr/local/zookeeper
KAFKA_INSTALL_DIR=/usr/local/kafka

NODE1=10.0.0.18
NODE2=10.0.0.28
NODE3=10.0.0.38

HOST=`hostname -I|awk '{print $1}'`
.  /etc/os-release


color () {
    RES_COL=60
    MOVE_TO_COL="echo -en \\033[${RES_COL}G"
    SETCOLOR_SUCCESS="echo -en \\033[1;32m"
    SETCOLOR_FAILURE="echo -en \\033[1;31m"
    SETCOLOR_WARNING="echo -en \\033[1;33m"
    SETCOLOR_NORMAL="echo -en \E[0m"
    echo -n "$1" && $MOVE_TO_COL
    echo -n "["
    if [ $2 = "success" -o $2 = "0" ] ;then
        ${SETCOLOR_SUCCESS}
        echo -n $"  OK  "    
    elif [ $2 = "failure" -o $2 = "1"  ] ;then 
        ${SETCOLOR_FAILURE}
        echo -n $"FAILED"
    else
        ${SETCOLOR_WARNING}
        echo -n $"WARNING"
    fi
    ${SETCOLOR_NORMAL}
    echo -n "]"
    echo 
}


install_jdk() {
    if [ $ID = 'centos' -o  $ID = 'rocky' ];then
        yum -y install java-1.8.0-openjdk-devel || { color "安装JDK失败!" 1; exit 1; }
    else
        apt update
        apt install openjdk-8-jdk -y || { color "安装JDK失败!" 1; exit 1; } 
    fi
    java -version
}

zk_myid () {
read -p "请输入node编号(默认为 1): " MYID

if [ -z "$MYID" ] ;then
    MYID=1
elif [[ ! "$MYID" =~ ^[0-9]+$ ]];then
    color  "请输入正确的node编号!" 1
    exit
else
    true
fi
}

install_zookeeper() {
    wget -P /usr/local/src/ $ZK_URL || { color  "下载zookeeper安装包失败!" 1 ;exit ; }
    tar xf /usr/local/src/${ZK_URL##*/} -C `dirname ${ZK_INSTALL_DIR}`
    ln -s /usr/local/apache-zookeeper-*-bin/ ${ZK_INSTALL_DIR}
    echo "PATH=${ZK_INSTALL_DIR}/bin:$PATH" >  /etc/profile.d/zookeeper.sh
    .  /etc/profile.d/zookeeper.sh
    mkdir -p ${ZK_INSTALL_DIR}/data 
    echo $MYID > ${ZK_INSTALL_DIR}/data/myid
    cat > ${ZK_INSTALL_DIR}/conf/zoo.cfg <<EOF
tickTime=2000
initLimit=10
syncLimit=5
dataDir=${ZK_INSTALL_DIR}/data
clientPort=2181
maxClientCnxns=128
autopurge.snapRetainCount=3
autopurge.purgeInterval=24
server.1=${NODE1}:2888:3888
server.2=${NODE2}:2888:3888
server.3=${NODE3}:2888:3888
EOF
   
	cat > /lib/systemd/system/zookeeper.service <<EOF
[Unit]
Description=zookeeper.service
After=network.target

[Service]
Type=forking
#Environment=${ZK_INSTALL_DIR}
ExecStart=${ZK_INSTALL_DIR}/bin/zkServer.sh start
ExecStop=${ZK_INSTALL_DIR}/bin/zkServer.sh stop
ExecReload=${ZK_INSTALL_DIR}/bin/zkServer.sh restart

[Install]
WantedBy=multi-user.target
EOF
    systemctl daemon-reload
    systemctl enable --now  zookeeper.service
    systemctl is-active zookeeper.service
    if [ $? -eq 0 ] ;then 
        color "zookeeper 安装成功!" 0  
    else 
        color "zookeeper 安装失败!" 1
        exit 1
    fi  
}


install_kafka(){
    wget -P /usr/local/src  $KAFKA_URL  || { color  "下载kafka安装包失败!" 1 ;exit ; }
    tar xf /usr/local/src/${KAFKA_URL##*/}  -C /usr/local/
    ln -s ${KAFKA_INSTALL_DIR}_*/ ${KAFKA_INSTALL_DIR}
    echo PATH=${KAFKA_INSTALL_DIR}/bin:'$PATH' > /etc/profile.d/kafka.sh
    . /etc/profile.d/kafka.sh
   
    cat > ${KAFKA_INSTALL_DIR}/config/server.properties <<EOF
broker.id=$MYID
listeners=PLAINTEXT://${HOST}:9092
log.dirs=${KAFKA_INSTALL_DIR}/data
num.partitions=1
log.retention.hours=168
zookeeper.connect=${NODE1}:2181,${NODE2}:2181,${NODE3}:2181
zookeeper.connection.timeout.ms=6000
EOF
    mkdir ${KAFKA_INSTALL_DIR}/data
	 
    cat > /lib/systemd/system/kafka.service <<EOF
[Unit]                                                                          
Description=Apache kafka
After=network.target

[Service]
Type=simple
#Environment=JAVA_HOME=/data/server/java
#PIDFile=${KAFKA_INSTALL_DIR}/kafka.pid
ExecStart=${KAFKA_INSTALL_DIR}/bin/kafka-server-start.sh  ${KAFKA_INSTALL_DIR}/config/server.properties
ExecStop=/bin/kill  -TERM \${MAINPID}
Restart=always
RestartSec=20

[Install]
WantedBy=multi-user.target

EOF
	systemctl daemon-reload
	systemctl enable --now kafka.service
	#kafka-server-start.sh -daemon ${KAFKA_INSTALL_DIR}/config/server.properties 
	systemctl is-active kafka.service
	if [ $? -eq 0 ] ;then 
        color "kafka 安装成功!" 0  
    else 
        color "kafka 安装失败!" 1
        exit 1
    fi    
}

zk_myid

install_jdk

install_zookeeper

install_kafka

执行脚本

[root@rocky18 ~]# bash install_kafka.sh 
请输入node编号(默认为 1): 1  
[root@rocky28 ~]# bash install_kafka.sh 
请输入node编号(默认为 1): 2  
[root@rocky38 ~]# bash install_kafka.sh 
请输入node编号(默认为 1): 3

测试

[root@rocky18 ~]# zkServer.sh status
/bin/java
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower

[root@rocky28 ~]# zkServer.sh status
/bin/java
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower

[root@rocky38 ~]# zkServer.sh status
/bin/java
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: leader


#上传zk图形化的客户端工具zooinspector-1.0-SNAPSHOT-pkg.tar
[root@rocky18 ~]# ls
install_kafka.sh  zooinspector-1.0-SNAPSHOT-pkg.tar
[root@rocky18 ~]# tar xf zooinspector-1.0-SNAPSHOT-pkg.tar 
[root@rocky18 ~]# ls
install_kafka.sh  zooinspector-1.0-SNAPSHOT  zooinspector-1.0-SNAPSHOT-pkg.tar
[root@rocky18 ~]# cd zooinspector-1.0-SNAPSHOT/bin/
[root@rocky18 bin]# ls
zooinspector.bat  zooinspector.sh

[root@rocky18 bin]# export DISPLAY=10.0.0.1:0
[root@rocky18 bin]# ./zooinspector.sh

在这里插入图片描述

#参数 --topic 指定 Topic 名，–partitions 指定分区数，–replication-factor 指定备份(副本)数
#创建 3分区，2副本
# kafka-topics.sh --create --topic wang --bootstrap-server 10.0.0.18:9092 --partitions 3 --replication-factor 2

[root@rocky18 bin]# ls /usr/local/kafka/data/
cleaner-offset-checkpoint    recovery-point-offset-checkpoint  wang-2
log-start-offset-checkpoint  replication-offset-checkpoint
meta.properties              wang-0
[root@rocky28 kafka]# ls /usr/local/kafka/data/
cleaner-offset-checkpoint    recovery-point-offset-checkpoint  wang-2
log-start-offset-checkpoint  replication-offset-checkpoint
meta.properties              wang-1
[root@rocky38 ~]# ls /usr/local/kafka/data/
cleaner-offset-checkpoint    recovery-point-offset-checkpoint  wang-1
log-start-offset-checkpoint  replication-offset-checkpoint
meta.properties              wang-0

#获取所有 Topic
[root@rocky28 kafka]# kafka-topics.sh --list --bootstrap-server 10.0.0.18:9092
wang
[root@rocky28 kafka]# kafka-topics.sh --list --bootstrap-server 10.0.0.18:9092
__consumer_offsets
wang

#验证 Topic 详情
# 状态说明：wang 有三个分区分别为0、1、2，分区0的leader是3 （broker.id），分区 0 有2 个副本，并且状态都为 lsr（ln-sync，表示可以参加选举成为 leader）。
[root@rocky38 ~]# kafka-topics.sh --describe --bootstrap-server 10.0.0.18:9092 --topic wang
Topic: wang	TopicId: do8QI_pgRhaYBqu49rQIDA	PartitionCount: 3	ReplicationFactor: 2 Configs: 
	Topic: wang	Partition: 0	Leader: 38	Replicas: 38,18	Isr: 38,18
	Topic: wang	Partition: 1	Leader: 28	Replicas: 28,38	Isr: 28,38
	Topic: wang	Partition: 2	Leader: 18	Replicas: 18,28	Isr: 18,28

#生产 Topic，默认可多人生产数据
[root@rocky38 ~]# kafka-console-producer.sh --topic wang --bootstrap-server 10.0.0.18:9092
>message1
>message2


#消费 Topic,默认可多人同时消费
[root@rocky28 kafka]# kafka-console-consumer.sh --topic wang --bootstrap-server 10.0.0.18:9092 --from-beginning
message1
message2

#消息者先生产消息，消费都后续才启动，也能收到之前生产的消息
#同一个消息在同一个group内的消费者只有被一个消费者消费，比如：共100条消息，在一个group内有A，B两个消费者，其中A消费50条，B消费另外的50条消息。从而实现负载均衡，不同group内的消费者则可以同时消费同一个消息

zookeeper

zookeeper配置文件说明

#默认配置可不做修改
[root@ubuntu1804 ~]#grep -v "#" /usr/local/zookeeper/conf/zoo.cfg
tickTime=2000 #"滴答时间"，用于配置Zookeeper中最小的时间单元长度，单位毫秒，是其它时间配置
的基础
initLimit=10 #初始化时间，包含启动和数据同步，其值是tickTime的倍数
syncLimit=5 #正常工作，心跳监测的时间间隔，其值是tickTime的倍数
dataDir=/tmp/zookeeper #配置Zookeeper服务存储数据的目录,基于安全,可以修改为
dataDir=/usr/local/zookeeper/data
#dataLogdir=/usr/local/zookeeper/logs #可以指定日志路径
clientPort=2181 #配置当前Zookeeper服务对外暴露的端口，用户客户端和服务端建立连接会话
autopurge.snapRetainCount=3 #3.4.0中的新增功能：启用后，ZooKeeper 自动清除功能,会将只保
留此最新3个快照和相应的事务日志,并分别保留在dataDir 和dataLogDir中，删除其余部分，默认值为3,
最小值为3
autopurge.purgeInterval=24 #3.4.0及之后版本，ZK提供了自动清理日志和快照文件的功能，这个参
数指定了清理频率，单位是小时，需要配置一个1或更大的整数，默认是 0，表示不开启自动清理功能

zk单机脚本

vim install_zookeeper_single_node.sh

#!/bin/bash


ZK_VERSION=3.8.3

ZK_URL=https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper-${ZK_VERSION}/apache-zookeeper-${ZK_VERSION}-bin.tar.gz
#ZK_URL="https://downloads.apache.org/zookeeper/stable/apache-zookeeper-${ZK_VERSION}-bin.tar.gz"

INSTALL_DIR=/usr/local/zookeeper


HOST=`hostname -I|awk '{print $1}'`

.  /etc/os-release

color () {
    RES_COL=60
    MOVE_TO_COL="echo -en \\033[${RES_COL}G"
    SETCOLOR_SUCCESS="echo -en \\033[1;32m"
    SETCOLOR_FAILURE="echo -en \\033[1;31m"
    SETCOLOR_WARNING="echo -en \\033[1;33m"
    SETCOLOR_NORMAL="echo -en \E[0m"
    echo -n "$1" && $MOVE_TO_COL
    echo -n "["
    if [ $2 = "success" -o $2 = "0" ] ;then
        ${SETCOLOR_SUCCESS}
        echo -n $"  OK  "    
    elif [ $2 = "failure" -o $2 = "1"  ] ;then 
        ${SETCOLOR_FAILURE}
        echo -n $"FAILED"
    else
        ${SETCOLOR_WARNING}
        echo -n $"WARNING"
    fi
    ${SETCOLOR_NORMAL}
    echo -n "]"
    echo 
}


install_jdk() {
    if [ $ID = 'centos' -o  $ID = 'rocky' ];then
        yum -y install java-1.8.0-openjdk-devel || { color "安装JDK失败!" 1; exit 1; }
    else
        apt update
        apt install openjdk-11-jdk -y || { color "安装JDK失败!" 1; exit 1; } 
        #apt install openjdk-8-jdk -y || { color "安装JDK失败!" 1; exit 1; } 
    fi
    java -version
}


install_zookeeper() {
    wget -P /usr/local/src/ --no-check-certificate $ZK_URL || { color  "下载失败!" 1 ;exit ; }
    tar xf /usr/local/src/${ZK_URL##*/} -C /usr/local
    ln -s /usr/local/apache-zookeeper-*-bin/ ${INSTALL_DIR}
    echo "PATH=${INSTALL_DIR}/bin:$PATH" >  /etc/profile.d/zookeeper.sh
    .  /etc/profile.d/zookeeper.sh
    mkdir -p ${INSTALL_DIR}/data 
    cat > ${INSTALL_DIR}/conf/zoo.cfg <<EOF
tickTime=2000
initLimit=10
syncLimit=5
dataDir=${INSTALL_DIR}/data
clientPort=2181
maxClientCnxns=128
autopurge.snapRetainCount=3
autopurge.purgeInterval=24
EOF
	cat > /lib/systemd/system/zookeeper.service <<EOF
[Unit]
Description=zookeeper.service
After=network.target

[Service]
Type=forking
#Environment=${INSTALL_DIR}
ExecStart=${INSTALL_DIR}/bin/zkServer.sh start
ExecStop=${INSTALL_DIR}/bin/zkServer.sh stop
ExecReload=${INSTALL_DIR}/bin/zkServer.sh restart

[Install]
WantedBy=multi-user.target
EOF
    systemctl daemon-reload
    systemctl enable --now  zookeeper.service
    systemctl is-active zookeeper.service
	if [ $? -eq 0 ] ;then 
        color "zookeeper 安装成功!" 0  
    else 
        color "zookeeper 安装失败!" 1
        exit 1
    fi   
}


install_jdk

install_zookeeper

zk测试

root@rocky8 ~]# zkServer.sh --help
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Usage: /usr/local/zookeeper/bin/zkServer.sh [--config <conf-dir>] {start|start-foreground|stop|version|restart|status|print-cmd}

[root@rocky8 ~]# zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: standalone

[root@rocky8 ~]# cat /usr/local/zookeeper/bin/../conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/local/zookeeper/data
clientPort=2181
maxClientCnxns=128
autopurge.snapRetainCount=3
autopurge.purgeInterval=24

[root@rocky8 ~]# tree /usr/local/zookeeper/data
/usr/local/zookeeper/data
├── version-2
│   └── snapshot.0
└── zookeeper_server.pid

1 directory, 2 files


[root@rocky8 ~]# ss -ntl
State             Recv-Q            Send-Q                       Local Address:Port                         Peer Address:Port            Process            
LISTEN            0                 128                                0.0.0.0:22                                0.0.0.0:*                                  
LISTEN            0                 128                                   [::]:22                                   [::]:*                                  
LISTEN            0                 50                                       *:2181                                    *:*                                  
LISTEN            0                 50                                       *:40585                                   *:*                                  
LISTEN            0                 50                                       *:8080                                    *:*                                  

[root@rocky8 ~]# ls /usr/local/zookeeper/bin/../logs
zookeeper--server-rocky8.wang.org.out

日志
root@rocky8 ~]# tail -f /usr/local/zookeeper/bin/../logs/zookeeper--server-rocky8.wang.org.out



#客户端连接

#连接本机
[root@rocky8 ~]# zkCli.sh
[zk: localhost:2181(CONNECTED) 0] 

#连接远程服务器
[root@rocky8 ~]# zkCli.sh -server 10.0.0.8:2181
[zk: 10.0.0.8:2181(CONNECTED) 0]

 #两个TAB 可以列出所有支持命令
[zk: localhost:2181(CONNECTED) 0] 
addWatch               addauth                close                  config                 connect                create                 
delete                 deleteall              delquota               get                    getAcl                 getAllChildrenNumber   
getEphemerals          history                listquota              ls                     printwatches           quit                   
reconfig               redo                   removewatches          set                    setAcl                 setquota               
stat                   sync                   version                whoami  

#ls查看目录
[zk: localhost:2181(CONNECTED) 0] ls /
[zookeeper]
[zk: localhost:2181(CONNECTED) 1] ls /zookeeper 
[config, quota]
[zk: localhost:2181(CONNECTED) 2] ls /zookeeper/config 
[]

# version版本
[zk: localhost:2181(CONNECTED) 3] version
ZooKeeper CLI version: 3.8.3-6ad6d364c7c0bcf0de452d54ebefa3058098ab56, built on 2023-10-05 10:34 UTC

#create 创建
[zk: localhost:2181(CONNECTED) 7] create /app1
Created /app1
[zk: localhost:2181(CONNECTED) 8] ls /
[app1, zookeeper]
[zk: localhost:2181(CONNECTED) 9] create /app1/m50
Created /app1/m50
# set增加修改数据 get查看数据
[zk: localhost:2181(CONNECTED) 12] set /app1/m50 lihong
[zk: localhost:2181(CONNECTED) 13] get /app1/m50
lihong

#get -s可以查看更详细的信息
[zk: localhost:2181(CONNECTED) 14] get -s /app1/m50
lihong
cZxid = 0xa
ctime = Sun Feb 18 15:26:26 CST 2024
mZxid = 0xd
mtime = Sun Feb 18 15:28:56 CST 2024
pZxid = 0xa
cversion = 0
dataVersion = 1
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 6
numChildren = 0

#delete删除数据
[zk: localhost:2181(CONNECTED) 15] ls /app1/m50 
[]
[zk: localhost:2181(CONNECTED) 16] delete /app1/m50
[zk: localhost:2181(CONNECTED) 17] ls /app1
[]

#stat 查看已知节点元数据
[zk: 10.0.0.8:2181(CONNECTED) 9] stat /zookeeper
zxid = Ox0 #节点创建时的zxid
ctime = Thu Jan 01 08:00:00 CST 1970 #节点创建时间
mzxid = Ox0 #节点最近一次更新时的zxid
mtime = Thu Jan 01 08:00:00 cST 1970 #节点最近一次更新的时间
pzxid = Ox0 #父节点创建时的zxid
cversion = -1 #子节点数据更新次数
dataversion = 0 #本节点数据更新次数
aclversion = o #节点ACL(授权信息)的更新次数
ephemera10wner = Ox0 #持久节点值为0，临时节点值为sessionid
dataLength = 0 #节点数据长度
numchi1dren = 1 #子节点个数

#quit退出
[zk: localhost:2181(CONNECTED) 19] quit

WATCHER::

WatchedEvent state:Closed type:None path:null
2024-02-18 15:38:02,782 [myid:] - INFO  [main:o.a.z.ZooKeeper@1232] - Session: 0x100006daa700004 closed
2024-02-18 15:38:02,782 [myid:] - INFO  [main-EventThread:o.a.z.ClientCnxn$EventThread@568] - EventThread shut down for session: 0x100006daa700004
2024-02-18 15:38:02,784 [myid:] - ERROR [main:o.a.z.u.ServiceUtils@48] - Exiting JVM with code 127