1. download kafka_2.11-0.10.1.0.tgz, upload to /usr/local
cd /usr/local
tar -zxvf kafka_2.11-0.10.1.0.tgz
mv kafka_2.11-0.10.1.0 kafka
2. config
vi ~/.bashrc
export KAFKA_HOME=/usr/local/kafka
export PATH= :$KAFKA_HOME/bin
source ~/.bashrc
vi kafka/config/server.properties
broker.id=0
log.dirs=/root/data/kafka/logs
zookeeper.connect=centos1:2181,centos2:2181,centos3:2181
3. add slf4j
download slf4j-1.7.21.zip, unzip it, and upload slf4j-nop-1.7.21.jar to /usr/local/kafka/libs
4. copy kafka to contos2, centos3
scp ~/.bashrc root@centos2:~/.bashrc
scp ~/.bashrc root@centos3:~/.bashrc
scp -r /usr/local/kafka root@centos2:/usr/local/
scp -r /usr/local/kafka root@centos3:/usr/local/
ssh contos2
source ~/.bashrc
vi /usr/local/kafka/config/server.properties
broker.id=1
ssh contos3
source ~/.bashrc
vi /usr/local/kafka/config/server.properties
broker.id=2
5 start kafka on centos1 and check if there is any error
cd /usr/local/kafka
bin/kafka-server-start.sh config/server.properties
error: can't allocate enough memory
solution:
vi bin/kafka-server-start.sh
modify: export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G" to export KAFKA_HEAP_OPTS="-Xmx1G -Xms128M"
modify bin/kafka-server-start.sh on centos2 and centos3
6. start zookeeper and kafka in backend on each server
ssh centos1:
zkServer.sh start
nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &
ssh centos2:
zkServer.sh start
nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &
ssh centos3:
zkServer.sh start
nohup /usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &
7 test for send and consumer messages
cd /usr/local/kafka/
bin/kafka-topics.sh --create --zookeeper centos1:2181,centos2:2181,centos3:2181 --replication-factor 1 --partitions 1 --topic testTopic
bin/kafka-console-producer.sh --broker-list centos1:9092,centos2:9092,centos3:9092 --topic testTopic
open another console window and login centos1:
cd /usr/local/kafka/
bin/kafka-console-consumer.sh --bootstrap-server centos1:9092,centos2:9092,centos3:9092 --topic testTopic --from-beginning
(if you print message in console window1, it will be received in console window2)