Kafka集群部署

Kafka集群部署

服务器基本信息

qfs服务器IP	安装服务	用户名	host
12.0.2.23	kafka_2.11-2.4.1	mppadmin	qfs004
12.0.2.24	kafka_2.11-2.4.1	mppadmin	qfs005
12.0.2.25	kafka_2.11-2.4.1	mppadmin	qfs006
12.0.2.26	kafka_2.11-2.4.1	mppadmin	qfs007
12.0.2.27	kafka_2.11-2.4.1	mppadmin	qfs008
12.0.2.28	kafka_2.11-2.4.1	mppadmin	qfs009

1 部署kafka集群

1.1 下载安装包，解压

tar -xzf kafka_2.11-2.4.1.tgz
cd kafka_2.11-2.4.1
mkdir kafka-logs
cd config
vim server.properties

1.2 修改配置文件

############################# Server Basics #############################
broker.id=0
############################# Socket Server Settings #############################
listeners=PLAINTEXT://12.0.2.23:9092
############################# Log Basics #############################
log.dirs=/home/mppadmin/kafka_2.11-2.4.1/kafka-logs
############################# Zookeeper #############################
zookeeper.connect=12.0.2.20:2181,12.0.2.21:2181,12.0.2.22:2181

1.3 修改后kafka_2.11-2.4.1拷贝至其他节点

其他节点修改server.properties

############################# Server Basics #############################
broker.id=1
############################# Socket Server Settings #############################
listeners=PLAINTEXT://12.0.2.24:9092

2 配置集群启动停止服务

每个节点配置kafka全局变量

# KAFKA_HOME author:liuyy
export KAFKA_HOME=/home/mppadmin/kafka_2.11-2.4.1
export PATH=$KAFKA_HOME/bin:$PATH

集群启动停止服务

#! /bin/bash

# Kafka代理节点地址
hosts=(qfs004 qfs005 qfs006 qfs007 qfs008 qfs009)

# 打印启动分布式脚本信息
mill=`date "+%N"`
tdate=`date "+%Y-%m-%d %H:%M:%S,${mill:0:3}"`

echo [$tdate] INFO [Kafka Cluster] begins to execute the $1 operation.

# 执行分布式开启命令    
function start()
{       
        for i in ${hosts[@]}
                do      
                        smill=`date "+%N"`
                        stdate=`date "+%Y-%m-%d %H:%M:%S,${smill:0:3}"`                        ssh mppadmin@$i "source /etc/profile;echo [$stdate] INFO [Kafka Broker $i] begins to execute the startup operation.;kafka-server-start.sh $KAFKA_HOME/config/server.properties>/dev/null" &
                        sleep 1
                done
}

# 执行分布式关闭命令    
function stop()
{
        for i in ${hosts[@]}
                do
                        smill=`date "+%N"`
                        stdate=`date "+%Y-%m-%d %H:%M:%S,${smill:0:3}"`
                        ssh mppadmin@$i "source /etc/profile;echo [$stdate] INFO [Kafka Broker $i] begins to execute the shutdown operation.;kafka-server-stop.sh>/dev/null;" &
                        sleep 1
                done
}
# 查看Kafka代理节点状态
function status()
{
        for i in ${hosts[@]}
                do
                        smill=`date "+%N"`
                        stdate=`date "+%Y-%m-%d %H:%M:%S,${smill:0:3}"`
                        ssh mppadmin@$i "source /etc/profile;echo [$stdate] INFO [Kafka Broker $i] status message is :;jps | grep Kafka;" &
                        sleep 1
                done
}

# 判断输入的Kafka命令参数是否有效
case "$1" in
    start)
        start
        ;;
    stop)
        stop
        ;;
    status)
        status
        ;;
    *)
        echo "Usage: $0 {start|stop|status}"
        RETVAL=1
esac

验证

[mppadmin@qfs004 ~]$ kafka-daemons.sh start
[2021-02-04 09:08:05,711] INFO [Kafka Cluster] begins to execute the start operation.
[2021-02-04 09:08:05,716] INFO [Kafka Broker qfs004] begins to execute the startup operation.
[2021-02-04 09:08:06,722] INFO [Kafka Broker qfs005] begins to execute the startup operation.
[2021-02-04 09:08:07,729] INFO [Kafka Broker qfs006] begins to execute the startup operation.
[2021-02-04 09:08:08,735] INFO [Kafka Broker qfs007] begins to execute the startup operation.
[2021-02-04 09:08:09,743] INFO [Kafka Broker qfs008] begins to execute the startup operation.
[2021-02-04 09:08:10,750] INFO [Kafka Broker qfs009] begins to execute the startup operation.

3 创建topic

[mppadmin@qfs004 ~]$  kafka-topics.sh --create -zookeeper qfs001:2181,qfs002:2181,qfs003:2181 --replication-factor 1 --partitions 12 --topic test
Created topic test.

4 查看topic信息

mppadmin@qfs004 ~]$ kafka-topics.sh --describe --zookeeper qfs001:2181,qfs002:2181,qfs003:2181 --topic test
Topic: test	PartitionCount: 12	ReplicationFactor: 1	Configs: 
	Topic: test	Partition: 0	Leader: 4	Replicas: 4	Isr: 4
	Topic: test	Partition: 1	Leader: 5	Replicas: 5	Isr: 5
	Topic: test	Partition: 2	Leader: 0	Replicas: 0	Isr: 0
	Topic: test	Partition: 3	Leader: 1	Replicas: 1	Isr: 1
	Topic: test	Partition: 4	Leader: 2	Replicas: 2	Isr: 2
	Topic: test	Partition: 5	Leader: 3	Replicas: 3	Isr: 3
	Topic: test	Partition: 6	Leader: 4	Replicas: 4	Isr: 4
	Topic: test	Partition: 7	Leader: 5	Replicas: 5	Isr: 5
	Topic: test	Partition: 8	Leader: 0	Replicas: 0	Isr: 0
	Topic: test	Partition: 9	Leader: 1	Replicas: 1	Isr: 1
	Topic: test	Partition: 10	Leader: 2	Replicas: 2	Isr: 2
	Topic: test	Partition: 11	Leader: 3	Replicas: 3	Isr: 3

5 查看所有topic信息

[mppadmin@qfs004 ~]$ kafka-topics.sh --zookeeper qfs001:2181,qfs002:2181,qfs001:2181 --list
test

6 消费test

kafka-console-consumer.sh --bootstrap-server 12.0.2.23:9092 --topic test --from-beginning