Kafka Tutorial

1. Install

Download Kafka from Kafka.apache.org{:target=“_blank”}, and extract to /opt/module.

[root@hadoop001 software]# tar xvf kafka_2.11-2.4.1.tgz  -C /opt/module/

2. Configuration

Go to $KAFKA_HOME/conf/ directory, change the following options in server.properties:

broker.id=0
listeners=PLAINTEXT://hadoop001:9092
log.dirs=/opt/module/kafka_2.11-2.4.1/data
zookeeper.connect=hadoop001:2181,hadoop002:2181,hadoop003:2181

Copy kafka folder to hadoop002/003

[root@hadoop001 module]# scp -r kafka_2.13-2.4.1/ hadoop002:`pwd`
[root@hadoop001 module]# scp -r kafka_2.13-2.4.1/ hadoop003:`pwd`

Change the broker.id/listeners in server.properties:
hadoop002:

broker.id=1
listeners=PLAINTEXT://hadoop002:9092

hadoop003:

broker.id=2
listeners=PLAINTEXT://hadoop003:9092

Set environment:

[root@hadoop001 kafka_2.11-2.4.1]# echo export KAFKA_HOME=`pwd` > /etc/profile
[root@hadoop001 kafka_2.11-2.4.1]# echo export PATH=$PATH:$KAFKA_HOME/bin > /etc/profile

3. Start Server

[root@hadoop001 kafka_2.11-2.4.1]# source /etc/profile
[root@hadoop001 kafka_2.11-2.4.1]# kafka-server-start.sh config/server.properties 2>&1 &
[1] 399641
[root@hadoop001 kafka_2.11-2.4.1]# [2022-06-29 15:10:02,391] INFO Registered kafka:type=kafka.Log4jController MBean (kafka.utils.Log4jControllerRegistration$)
[2022-06-29 15:10:02,825] INFO Registered signal handlers for TERM, INT, HUP (org.apache.kafka.common.utils.LoggingSignalHandler)
[2022-06-29 15:10:02,825] INFO starting (kafka.server.KafkaServer)
[2022-06-29 15:10:02,831] INFO Connecting to zookeeper on hadoop001:2181,hadoop002:2181,hadoop003:2181 (kafka.server.KafkaServer)
[2022-06-29 15:10:02,856] INFO [ZooKeeperClient Kafka server] Initializing a new session to hadoop001:2181,hadoop002:2181,hadoop003:2181. (kafka.zookeeper.ZooKeeperClient)
...
[2022-06-29 15:10:04,185] INFO Kafka version: 2.4.1 (org.apache.kafka.common.utils.AppInfoParser)
[2022-06-29 15:10:04,185] INFO Kafka commitId: c57222ae8cd7866b (org.apache.kafka.common.utils.AppInfoParser)
[2022-06-29