一文吃透企业级elk技术栈：4. kafka 集群部署

四、kafka 集群部署

Kafka 依赖Zookeeper环境运行，所以需要先搭建zk集群

1、Zookeeper集群部署

# 下载安装包
wget  https://dlcdn.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0-bin.tar.gz --no-check-certificate

# 上传到其他节点
scp apache-zookeeper-3.7.0-bin.tar.gz  10.10.8.165:/root
scp apache-zookeeper-3.7.0-bin.tar.gz  10.10.8.166:/root

# 安装
tar xf apache-zookeeper-3.7.0-bin.tar.gz
mv apache-zookeeper-3.7.0-bin /usr/local/zookeeper


# 配置
vim  /usr/local/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/usr/local/zookeeper/data
clientPort=2181
autopurge.snapRetainCount=3
autopurge.purgeInterval=1
server.1=10.10.8.164:2888:3888
server.2=10.10.8.165:2888:3888
server.3=10.10.8.166:2888:3888

# 创建数据目录并指定myid
mkdir /usr/local/zookeeper/data
echo 1 >/usr/local/zookeeper/data/myid # 每个机器的id不同 可以 echo为  1  2  3

# 启动zk
/usr/local/zookeeper/bin/zkServer.sh start

# 查看状态
/usr/local/zookeeper/bin/zkServer.sh status  //一个节点显示leader，其他节点显示follower则成功
Mode: leader
Mode: follower

# 验证
/usr/local/zookeeper/bin/zkCli.sh # 连接zk
create /elk  # 创建索引
ls /         # 查看索引

2、kafka 集群部署

# 下载安装包
wget https://downloads.apache.org/kafka/3.1.0/kafka_2.12-3.1.0.tgz

# 上传到其他节点
scp kafka_2.12-3.1.0.tgz  10.10.8.165:/root
scp kafka_2.12-3.1.0.tgz  10.10.8.166:/root


# 安装
tar xf  kafka_2.12-3.1.0.tgz
mv kafka_2.12-3.1.0  /usr/local/kafka


# 配置
vim  /usr/local/kafka/bin/kafka-server-start.sh
    export KAFKA_HEAP_OPTS="-Xmx256M -Xms256M" # 修改最大内存占用


vim /usr/local/kafka/config/server.properties
# kafka 本机监听地址
listeners=PLAINTEXT://10.10.8.164:9092 
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
# 日志保留时间
num.partitions=1  
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
# zk 连接信息
zookeeper.connect=10.10.8.164:2181,10.10.8.165:2181,10.10.8.166:2181 
zookeeper.connection.timeout.ms=60000
group.initial.rebalance.delay.ms=0

# 使用systemctl管理
vim /usr/lib/systemd/system/kafka.service
[Unit]
Description=kafka
After=network.target
[Service]
Type=simple
ExecStart=/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
User=root
[Install]
WantedBy=multi-user.target
# Kafka启动
systemctl enable kafka
systemctl start kafka
# 验证
netstat -lntp|grep  9092
ps -ef|grep  kafka
# 创建topic，创建成功说明kafka集群搭建成功
/usr/local/kafka/bin/kafka-topics.sh --bootstrap-server  10.10.8.164:9092 --create  --replication-factor 2 --partitions 2 --topic elktest
# 验证
/usr/local/kafka/bin/kafka-topics.sh --describe --bootstrap-server 10.10.8.165:9092 --topic elktest