Kafka组件部署
0.准备工作
- 配置主机名和ip
# master 查看ip
hostname master && bash
echo master > /etc/hostname
ip a
# slave1 查看ip
hostname master && bash
echo slave1 > /etc/hostname
ip a
# slave2 查看ip
hostname master && bash
echo slave2 > /etc/hostname
ip a
# 配置ip映射
vi /etc/hosts
# 例如,ip不代表你本机ip,记得更换
# 三台都需要配置hosts文件
192.168.10.3 master
192.168.10.4 slave1
192.168.10.5 slave2
- 配置免密
# 查看ssh进程
ps - ef | grep ssh
# master
# 生成密钥ssh-keygen -t rsa 一直回车
# 复制key文件ssh-copy-id 输入yes 输入密码
ssh-keygen -t rsa
ssh-copy-id master
ssh-copy-id slave1
ssh-copy-id slave2
# slave1
ssh-keygen -t rsa
ssh-copy-id master
ssh-copy-id slave1
ssh-copy-id slave2
# slave2
ssh-keygen -t rsa
ssh-copy-id master
ssh-copy-id slave1
ssh-copy-id slave2
- 测试免密
ssh slave1
- 关闭防火墙
# master slave1 slave2
systemctl stop firewalld
systemctl disable firewalld
- 安装java
- /root/.bashrc只对当前用户有效,/etc/profile是对全部用户有效
tar -zxvf /opt/software/jdk-8u161-linux-x64.tar.gz -C /usr/local/src
mv /usr/local/src/jdk1.8.0_65 /usr/local/src/java
vi /root/.bashrc
export JAVA_HOME=/usr/local/src/java
export PATH=$PATH:$JAVA_HOME/bin
scp -r /usr/local/src/java slave1:/usr/local/src
scp -r /usr/local/src/java slave2:/usr/local/src
scp /root/.bashrc slave1:/root
scp /root/.bashrc slave2:/root
# master
source /root/.bashrc
# slave1
source /root/.bashrc
# slave2
source /root/.bashrc
软件名称 | 软件包名 |
---|---|
Zookeeper | zookeeper-3.4.8.tar.gz |
Kafka | kafka_2.11-1.0.0.tgz |
jdk | jdk-8u65-linux-x64.tar.gz |
CentOS | CentOS 7.4(mini 英文版本) |
1.安装zookeeper 并配置 启动
-
安装过了不用安装!!!!
-
需安装 Zookeeper 组件具体要求同 Zookeeper 任务要求
- 解压安装包到指定目录
tar -zxvf /opt/software/zookeeper-3.4.8.tar.gz -C /usr/local/src
- 改名
cd /usr/local/src/
mv zookeeper-3.4.8 zookeeper
- 修改zoo.cfg配置文件
cd zookeeper/conf
mv zoo_sample.cfg zoo.cfg
vi zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
clientPort=2181
dataDir=/usr/local/src/zookeeper/data
dataLogDir=/usr/local/src/zookeeper/logs
server.1=master:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888
- 创建data和logs目录
mkdir -p /usr/local/src/zookeeper/{data,logs}
- 创建myid文件
echo 1 > /usr/local/src/zookeeper/data/myid
- 配置环境zookeeper变量
vi /root/.bashrc
export ZK_HOME=/usr/local/src/zookeeper
export PATH=$PATH:$ZK_HOME/bin
source /root/.bashrc
chown -R root:root usr/local/src/zookeeper
- 分发zookeeper和profile文件到其他集群
## master
scp -r /usr/local/src/zookeeper slave1:/usr/local/src
scp -r /usr/local/src/zookeeper slave2:/usr/local/src
scp /root/.bashrc slave1:/root
scp /root/.bashrc slave2:/root
- 更改slave1的myid文件
# slave1
echo 2 > /usr/local/src/zookeeper/data/myid
source /root/.bashrc
- 更改slave2的myid文件
# slave2
echo 3 > /usr/local/src/zookeeper/data/myid
source /root/.bashrc
- 进入zookeeper目录关闭防火墙并启动zookeeper
# master
cd /usr/local/src/zookeeper
zkServer.sh start
# slave1
cd /usr/local/src/zookeeper
zkServer.sh start
# slave2
cd /usr/local/src/zookeeper
zkServer.sh start
- 查看是否成功并截图保存结果
# master
zkServer.sh status
# slave1
zkServer.sh status
# slave2
zkServer.sh status
2.安装Kafka
- 解压 Kafka 安装包到“/usr/local/src”路径
tar -zxvf /opt/software/kafka_2.11-1.0.0.gz -C /usr/local/src
- 并修改解压后文件夹名为 kafka,截图并保存结果;并修改解压后文件夹名为 kafka
cd /usr/local/src
mv kafka_2.11-1.0.0 kafka
- 截图并保存结果;
ll
3.配置kafka环境变量
- 设置 Kafka 环境变量,并使环境变量只对当前 root 用户生效,截图并保存 结果;
vi /root/.bashrc
export KAFKA_HOME=/usr/local/src/kafka
export PATH=$PATH:$KAFKA_HOME/bin
# :wq! 保存退出并source
source /root/.bashrc
4. 修改kafka相应文件并截图保存
- 修改master并截图
# master
# /zookeeper 快速查找,n向下查找,N向上查找,查到后直接 i 进入编辑模式
vi kafka/config/server.properties
broker.id=0
zookeeper.connect=master:2181,slave1:2181,slave2:2181
- 分发
scp -r /usr/local/src/kafka slave1:/usr/local/src
scp -r /usr/local/src/kafka slave2:/usr/local/src
scp /root/.bashrc slave1:/root
scp /root/.bashrc slave2:/root
- 修改slave1 slave2并截图保存
# 修改slave1和slave2的broker.id
# slave1
vi /usr/local/src/kafka/config/server.properties
broker.id=1
#保存退出
source /root/.bashrc
# slave2
vi /usr/local/src/kafka/config/server.properties
broker.id=2
#保存退出
source /root/.bashrc
5.启动kafka并截图保存结果
# master
kafka-server-start.sh -daemon /usr/local/src/kafka/config/server.properties
# slave1
kafka-server-start.sh /usr/local/src/kafka/config/server.properties
# slave2
kafka-server-start.sh /usr/local/src/kafka/config/server.properties
6.创建指定topic并截图保存结果
# --topic 后跟 topic的名字
kafka-topics.sh --create --zookeeper master:2181,slave1:2181,slave2:2181 --replication-factor 2 --topic hello --partitions 1
7.查看所有topic信息并截图保存结果
kafka-topics.sh --list --zookeeper master:2181,slave1:2181,slave2:2181
8.启动指定生产者(producer)并截图并保存结果;
# --topic 后面为tpoic的名字
kafka-console-producer.sh --broker-list master:9092,slave1:9091,slave2:9092 --topic hello
9. 启动消费者(consumer),并截图并保存结果
# --topic 后面为tpoic的名字
kafka-console-consumer.sh --bootstrap-server master:9092,slave1:9091,slave2:9092 --topic hello --from-beginning
10.测试生产者(producer),并截图并保存结果;
# producer随便输入点东西
11.测试消费者(consumer),并截图并保存结果。
# consumer可以看到输出