kafka安装

# 环境：
java8,centos7
192.168.147.152 node3
192.168.147.153 node4
192.168.147.154 node5

一、下载

官方下载地址：https://www.apache.org/dyn/closer.cgi?path=/kafka/3.1.0/kafka_2.13-3.1.0.tgz
下载：https://dlcdn.apache.org/kafka/3.1.0/kafka_2.13-3.1.0.tgz

二、上传安装包到/usr/local/kafka

node5,node4,node3都上传至/usr/local/kafka/

三、解压

# node5,node4,node3都执行
# tar -zxvf kafka_2.13-3.1.0.tgz

四、配置环境变量

# vim /etc/profile
export KAFKA_HOME=/usr/local/kafka/kafka_2.13-3.1.0
export PATH=$PATH:$KAFKA_HOME/bin
# source /etc/profile

五、修改配置文件

# 注意配置主机名
# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.147.150 node1
192.168.147.151 node2
192.168.147.152 node3
192.168.147.153 node4
192.168.147.154 node5

# 配置第一台node5
# 进入配置
cd /usr/local/kafka/kafka_2.13-3.1.0/config
# 创建数据目录
mkdir -p /usr/local/kafka/kafka_2.13-3.1.0/logs
# vim server.properties
# cat server.properties |grep -v '#'
broker.id=0
listeners=PLAINTEXT://node5:9092
advertised.listeners=PLAINTEXT://node5:9092
log.dirs=/usr/local/kafka/kafka_2.13-3.1.0/logs
num.partitions=2
zookeeper.connect=node3:2181,node4:2181,node5:2181
# 配置第二台node4
broker.id=1
listeners=PLAINTEXT://node4:9092
advertised.listeners=PLAINTEXT://node4:9092
log.dirs=/usr/local/kafka/kafka_2.13-3.1.0/logs
num.partitions=2
zookeeper.connect=node3:2181,node4:2181,node5:2181
# 配置第三台node3
broker.id=2
listeners=PLAINTEXT://node3:9092
advertised.listeners=PLAINTEXT://node3:9092
log.dirs=/usr/local/kafka/kafka_2.13-3.1.0/logs
num.partitions=2
zookeeper.connect=node3:2181,node4:2181,node5:2181

六、关闭node5/node4/node3防火墙或开放端口

1.关闭防火墙（生产环境禁止操作，由运维配置开放端口）

systemctl stop firewalld
systemctl disable firewalld

2.CentOS7开放端口

# 开放端口2181 2888 3888 9092 
firewall-cmd --zone=public --add-port=2181/tcp --permanent
#命令含义：
#    --zone #作用域
#    --add-port=2181/tcp  #添加端口，格式为：端口/通讯协议
#    --permanent  #永久生效，没有此参数重启后失效
# 重启防火墙
firewall-cmd --reload
# 查看端口
netstat -ntlp 

七、启动

分别启动node3,node4,node5

cd /usr/local/kafka/kafka_2.13-3.1.0/config
# kafka-server-start.sh -daemon server.properties

八、测试

1.在/usr/local/kafka/kafka_2.13-3.1.0/bin/目录下封装默认脚本，简化参数输入

vim default-kafka-topics-create.sh

echo "--topic $1 --partitions $2 --replication-factor $3"
if [ -z $1 ]; then
  echo "topic not allow empty "
  exit 1
fi
partition=3
repfactor=3
if [ -n $2 ]; then
  partition=$2
fi
if [ -n $3 ]; then
  repfactor=$3
fi
echo "kafka-topics.sh --bootstrap-server node3:9092,node4:9092,node5:9092 --create --topic $1 --partitions $partition --replication-factor $repfactor"
kafka-topics.sh --bootstrap-server node3:9092,node4:9092,node5:9092 --create --topic $1 --partitions $partition --replication-factor $repfactor

vim default-kafka-console-producer.sh

if [ -z "$1" ];then
  echo "topic not allow empty"
  exit 1
fi
echo "kafka-console-producer.sh --bootstrap-server node3:9092,node4:9092,node5:9092 --topic $1"
kafka-console-producer.sh --bootstrap-server node3:9092,node4:9092,node5:9092 --topic $1

vim default-kafka-console-consumer.sh

echo "--topic $1 --partition $2 --group $3"
if [ -z "$1" ]; then
   echo "topic not allow empty"
   exit 1
else 
   echo "consumer topic:$1"
fi

if [ -n "$2" ]; then
   if [ -n "$3" ]; then
      echo "kafka-console-consumer.sh --bootstrap-server node3:9092,node4:9092,node5:9092 --topic $1 --partition $2 --group $3 --property print.timestamp=true --property print.key=true --property  print.value=true --property  key.separator=, --property print.offset=true  --property  print.partition=true --property  print.headers=true"
      kafka-console-consumer.sh --bootstrap-server node3:9092,node4:9092,node5:9092 --topic $1 --partition $2 --group $3 --property print.timestamp=true --property print.key=true --property  print.value=true --property  key.separator=, --property print.offset=true  --property  print.partition=true --property  print.headers=true
   else
      echo "kafka-console-consumer.sh --bootstrap-server node3:9092,node4:9092,node5:9092  --topic $1 --partition $2 --property print.timestamp=true --property print.key=true --property  print.value=true --property  key.separator=, --property print.offset=true  --property  print.partition=true --property  print.headers=true"
      kafka-console-consumer.sh --bootstrap-server node3:9092,node4:9092,node5:9092 --topic $1 --partition $2 --property print.timestamp=true --property print.key=true --property  print.value=true --property  key.separator=, --property print.offset=true  --property  print.partition=true --property  print.headers=true
   fi
else
   echo "kafka-console-consumer.sh --bootstrap-server node3:9092,node4:9092,node5:9092  --topic $1 --property print.timestamp=true --property print.key=true --property  print.value=true --property  key.separator=, --property print.offset=true  --property  print.partition=true --property  print.headers=true"
   kafka-console-consumer.sh --bootstrap-server node3:9092,node4:9092,node5:9092 --topic $1 --property print.timestamp=true --property print.key=true --property  print.value=true --property  key.separator=, --property print.offset=true  --property  print.partition=true --property  print.headers=true
fi

2.验证