在linux搭建kafka集群（shell脚本搭建）

 第一步：配置jdk

1、将jdk的安装包下载到指定目录/opt/software，其中的文件路径自行修改

#!/bin/sh
#安装jdk

PACKAGE_DIR="/opt/software"
cd $PACKAGE_DIR

tar -xf jdk-8u361-linux-x64.tar.gz

#添加JDK的环境变量
sed -i '$a #jdk' /etc/profile
sed -i '$a export JAVA_HOME=/opt/software/jdk1.8.0_361' /etc/profile
sed -i '$a export PATH=$JAVA_HOME/bin:$PATH' /etc/profile
sed -i '$a export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar' /etc/profile
source /etc/profile

#echo \ #jdk \  >> /etc/profile
#echo export JAVA_HOME=/opt/software/jdk1.8.0_361 >> /etc/profile
#echo export PATH=$JAVA_HOME/bin:$PATH >> /etc/profile
#echo export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar >> /etc/profile
source /etc/profile

echo "node1 jdk配置完成。。。"

#配置节点jdk
sh /root/install/jdk/node_install.sh

#node_install.sh
#bin/bash
PACKAGE_DIR="/opt/software"
nodes=(
"test002"
"test003"
)

for node in ${nodes[@]};
do
ssh ${node} "mkdir -p $PACKAGE_DIR"
scp -r $PACKAGE_DIR/jdk1.8.0_361/ root@${node}:/opt/software/
ssh $node "source /etc/profile
echo \" #jdk\" >> /etc/profile
echo export JAVA_HOME=/opt/software/jdk1.8.0_361 >> /etc/profile
echo export PATH=\\\$JAVA_HOME/bin:\\\$PATH >> /etc/profile
echo export CLASSPATH=.:\\\$JAVA_HOME/lib/dt.jar:\\\$JAVA_HOME/lib/tools.jar >> /etc/profile
"

source /etc/profile
echo "${node} jdk配置完成"

done

2、检查jdk是否安装成功

#查看java是否安装完成
java -version

第二步：配置zookeeper

同上，将zookeeper的安装包下载到指定目录/opt/software，其中的文件路径自行修改，配置变量根据自己的实际情况修改

#!/bin/sh
#安装zk

PACKAGE_DIR="/opt/software"
cd $PACKAGE_DIR
tar -xf apache-zookeeper-3.6.4-bin.tar.gz
rm -rf apache-zookeeper-3.6.4-bin.tar.gz
mv apache-zookeeper-3.6.4-bin zookeeper

cd $PACKAGE_DIR/zookeeper/conf
cp zoo_sample.cfg zoo.cfg
sed -i 's#dataDir=/tmp/zookeeper#dataDir=/opt/software/zookeeper/data#g' /opt/software/zookeeper/conf/zoo.cfg
sed -i '$a server.1=192.168.44.129:2888:3888' /opt/software/zookeeper/conf/zoo.cfg
sed -i '$a server.2=192.168.44.128:2888:3888' /opt/software/zookeeper/conf/zoo.cfg
sed -i '$a server.3=192.168.44.131:2888:3888' /opt/software/zookeeper/conf/zoo.cfg

mkdir -p $PACKAGE_DIR/zookeeper/data
cd $PACKAGE_DIR/zookeeper/data

# 生成节点号myid文件
echo 1 > myid

#添加zk的环境变量
sed -i '$a #zookeeper' /etc/profile
sed -i '$a export ZOOKEEPER_HOME=/opt/software/zookeeper' /etc/profile
sed -i '$a export PATH=$PATH:$ZOOKEEPER_HOME/bin:$ZOOKEEPER_HOME/conf' /etc/profile
source /etc/profile

#echo \" #zookeeper\" >> /etc/profile
#echo export ZOOKEEPER_HOME=/opt/software/zookeeper >> /etc/profile
#echo export PATH=\\\$PATH:\\\$ZOOKEEPER_HOME/bin:\\\$ZOOKEEPER_HOME/conf >> /etc/profile

echo "node1配置完成"
/opt/software/zookeeper/bin/zkServer.sh start

sh /root/install/zookeeper/node_install.sh

#!/bin/sh
#安装节点zk

PACKAGE_DIR="/opt/software"
nodes=(
"test002"
"test003"
)

for node in ${nodes[@]};
do
ssh ${node} "mkdir -p $PACKAGE_DIR"
scp -r $PACKAGE_DIR/zookeeper/ root@${node}:/opt/software/
ssh $node "source /etc/profile
echo \" #zookeeper\" >> /etc/profile
echo export ZOOKEEPER_HOME=/opt/software/zookeeper >> /etc/profile
echo export PATH=\\\$PATH:\\\$ZOOKEEPER_HOME/bin:\\\$ZOOKEEPER_HOME/conf >> /etc/profile
"
echo "${node}配置完成"

done

ssh ${nodes[0]} "sed -i 's#1#2#g' /opt/software/zookeeper/data/myid"
ssh ${nodes[1]} "sed -i 's#1#3#g' /opt/software/zookeeper/data/myid"

#启动zookeeper
/opt/software/zookeeper/bin/zkServer.sh start
/opt/software/zookeeper/bin/zkServer.sh status

#连接zookeeper的客户端
bin/zkCli.sh -server 192.168.44.129:2181
ls /
ls /brokers
ls /brokers/ids

第三步：配置kafka

和前两个步骤一样，将zookeeper的安装包下载到指定目录/opt/software，其中的文件路径自行修改，配置变量根据自己的实际情况修改

#!/bin/sh
#安装kafka

PACKAGE_DIR="/opt/software"
cd $PACKAGE_DIR
tar -xf kafka_2.13-3.4.0.tgz
#rm -rf kafka_2.13-3.4.0.tgz
mv kafka_2.13-3.4.0 kafka

cd $PACKAGE_DIR/kafka/config
cp server.properties server.properties.bak

#修改kafka的配置文件
#sed -i 's|broker.id=0|broker.id=1|g' /opt/software/kafka/config/server.properties
sed -i 's|log.dirs=/tmp/kafka-logs|log.dirs=/opt/software/kafka/kafka-logs|g' /opt/software/kafka/config/server.properties
sed -i 's|zookeeper.connect=localhost:2181|zookeeper.connect=192.168.44.129:2181,192.168.44.128:2181,192.168.44.131:2181|g' /opt/software/kafka/config/server.properties
sed -i 's|#listeners=|listeners=|g' /opt/software/kafka/config/server.properties
sed -i 's|#advertised.listeners=PLAINTEXT://your.host.name:9092|advertised.listeners=PLAINTEXT://192.168.44.129:9092|g' /opt/software/kafka/config/server.properties
sed -i '$a delete.topic.enble=true' /opt/software/kafka/config/server.properties

#添加kafka的环境变量
sed -i '$a #kafka' /etc/profile
sed -i '$a export KAFKA_HOME=/opt/software/kafka' /etc/profile
# sed -i '$a export PATH=KAFKA_HOME/bin:$PATH' /etc/profile
source /etc/profile

echo "kafka-node1配置完成"

scp -r $PACKAGE_DIR/kafka/ root@test002:/opt/software/
ssh test002 " source /etc/profile
echo \" #kafka\" >> /etc/profile
echo export KAFKA_HOME=/opt/software/kafka >> /etc/profile
echo export PATH=KAFKA_HOME/bin:\\\$PATH >> /etc/profile
sed -i 's|broker.id=0|broker.id=1|g' /opt/software/kafka/config/server.properties
sed -i 's|advertised.listeners=PLAINTEXT://192.168.44.129:9092|advertised.listeners=PLAINTEXT://192.168.44.128:9092|g' /opt/software/kafka/config/server.properties
"

echo "kafka-node2配置完成"


scp -r $PACKAGE_DIR/kafka/ root@test003:/opt/software/
ssh test003 " source /etc/profile
echo \" #kafka\" >> /etc/profile
echo export KAFKA_HOME=/opt/software/kafka >> /etc/profile
echo export PATH=KAFKA_HOME/bin:\\\$PATH >> /etc/profile
sed -i 's|broker.id=0|broker.id=2|g' /opt/software/kafka/config/server.properties
sed -i 's|advertised.listeners=PLAINTEXT://192.168.44.129:9092|advertised.listeners=PLAINTEXT://192.168.44.131:9092|g' /opt/software/kafka/config/server.properties
"
echo "kafka-node3配置完成"

#启动kafka
cd /opt/software/kafka
./bin/kafka-server-start.sh config/server.properties &

# 进入kafka安装目录bin,创建topic
cd /opt/software/kafka/bin/
./kafka-topics.sh --create --topic test --bootstrap-server 192.168.44.129:9092

查看topic
./kafka-topics.sh --list --topic test --bootstrap-server 192.168.44.129:9092

# 往topic发送消息
./kafka-console-producer.sh --broker-list 192.168.44.129:9092 --topic test

# 消费topic test
./kafka-console-consumer.sh --bootstrap-server 192.168.44.129:9092 --topic test --from-beginning

./kafka-topics.sh --delete --topic test --bootstrap-server 192.168.44.129:9092