Kafka: ------ 单机环境搭建、集群搭建

Kafka环境搭建

单机环境

- 安装JDK,配置JAVA_HOME

[root@CentOS ~]# rpm -ivh jdk-8u191-linux-x64.rpm 
warning: jdk-8u191-linux-x64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Preparing...                ########################################### [100%]
   1:jdk1.8                 ########################################### [100%]
Unpacking JAR files...
	tools.jar...
	plugin.jar...
	javaws.jar...
	deploy.jar...
	rt.jar...
	jsse.jar...
	charsets.jar...
	localedata.jar...
[root@CentOS ~]# vi .bashrc 
JAVA_HOME=/usr/java/latest
CLASSPATH=.
PATH=$PATH:$JAVA_HOME/bin
export JAVA_HOME
export CLASSPATH
export PATH
[root@CentOS ~]# vi .bashrc 

- 配置主机名为CentOS

[root@CentOS ~]# vim /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=CentOS

- 配置主机名和IP映射

[root@CentOS ~]# ifconfig 
eth0      Link encap:Ethernet  HWaddr 00:0C:29:D3:EA:13  
          inet addr:192.168.52.129  Bcast:192.168.52.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fed3:ea13/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:641 errors:0 dropped:0 overruns:0 frame:0
          TX packets:379 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:56374 (55.0 KiB)  TX bytes:57374 (56.0 KiB)

lo        Link encap:Local Loopback  
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)
[root@CentOS ~]# vi /etc/hosts      
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.52.129 CentOS

- 关闭防火墙

[root@CentOS ~]# service iptables stop
iptables: Setting chains to policy ACCEPT: filter          [  OK  ]
iptables: Flushing firewall rules:                         [  OK  ]
iptables: Unloading modules:                               [  OK  ]
[root@CentOS ~]# chkconfig iptables off
[root@CentOS ~]# chkconfig --list | grep iptables
iptables       	0:关闭	1:关闭	2:关闭	3:关闭	4:关闭	5:关闭	6:关闭

- 安装配置Zookeeper

[root@CentOS ~]# tar -zxf zookeeper-3.4.6.tar.gz -C /usr/
[root@CentOS ~]# cd /usr/zookeeper-3.4.6/
[root@CentOS zookeeper-3.4.6]# cp conf/zoo_sample.cfg conf/zoo.cfg
[root@CentOS zookeeper-3.4.6]# vi conf/zoo.cfg 
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/root/zkdata
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
[root@CentOS ~]# mkdir /root/zkdata

- 启动zookeeper服务

[root@CentOS zookeeper-3.4.6]# ./bin/zkServer.sh 
JMX enabled by default
Using config: /usr/zookeeper-3.4.6/bin/../conf/zoo.cfg
Usage: ./bin/zkServer.sh {start|start-foreground|stop|restart|status|upgrade|print-cmd}
[root@CentOS zookeeper-3.4.6]# ./bin/zkServer.sh start zoo.cfg
JMX enabled by default
Using config: /usr/zookeeper-3.4.6/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@CentOS zookeeper-3.4.6]# 
[root@CentOS zookeeper-3.4.6]# ./bin/zkServer.sh status zoo.cfg
JMX enabled by default
Using config: /usr/zookeeper-3.4.6/bin/../conf/zoo.cfg
Mode: standalone
[root@CentOS zookeeper-3.4.6]# jps
1778 Jps
1733 QuorumPeerMain

- 安装配置Kafka单机

[root@CentOS ~]# tar -zxf kafka_2.11-2.2.0.tgz -C /usr/
[root@CentOS ~]# cd /usr/kafka_2.11-2.2.0/
[root@CentOS kafka_2.11-2.2.0]# vi config/server.properties 
############################# Server Basics #############################

# The id of the broker. This must be set to a unique integer for each broker.
broker.id=0

############################# Socket Server Settings #############################

listeners=PLAINTEXT://CentOS:9092

############################# Log Basics #############################

log.dirs=/usr/kafka-logs

############################# Log Retention Policy #############################

log.retention.hours=168

############################# Zookeeper #############################

zookeeper.connect=CentOS:2181


启动kafka

[root@CentOS kafka_2.11-2.2.0]# ./bin/kafka-server-start.sh -daemon config/server.properties 
[root@CentOS kafka_2.11-2.2.0]#./bin/kafka-topics.sh --bootstrap-server Centos:9092 \ 

- 测试Kafka服务 创建Topic

[root@CentOS kafka_2.11-2.2.0]# ./bin/kafka-topics.sh 
                                                --bootstrap-server CentOS:9092 
                                                --create --topic topic01 
                                                --partitions 1 
                                                --replication-factor 1

kafka自2.2.0版本以后,Toipic的管理使用的的是--bootstrap-server不在使用--zookeeper--partitions:指定分区数、--replication-factor指定副本因子数,该副本因子不能大于可用的broker节点的个数

查看Topic列表

[root@CentOS kafka_2.11-2.2.0]# ./bin/kafka-topics.sh 
                                                --bootstrap-server CentOS:9092 
                                                --list 
topic01

查看Topic详情

[root@CentOS kafka_2.11-2.2.0]# ./bin/kafka-topics.sh --bootstrap-server CentOS:9092 --describe --topic topic01
Topic:topic01   PartitionCount:3        ReplicationFactor:1     Configs:segment.bytes=1073741824
        Topic: topic01  Partition: 0    Leader: 0       Replicas: 0     Isr: 0
        Topic: topic01  Partition: 1    Leader: 0       Replicas: 0     Isr: 0
        Topic: topic01  Partition: 2    Leader: 0       Replicas: 0     Isr: 0

segment.bytes:Kafka底层在存储分区的文件的时候是按照段落存储的,也就是某个分区的文件达到1GB(1073741824 bytes)的时候,系统会生成新的段落,这种设计有助于Broker节点索引文件。Replicas:表示副本集成员broker-id,Isr:表示处于同步中的正常副本集全称(In Synch Replicate)

集群环境

- 安装JDK,配置JAVA_HOME

[root@CentOSX ~]# rpm -ivh jdk-8u191-linux-x64.rpm 
warning: jdk-8u191-linux-x64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Preparing...                ########################################### [100%]
   1:jdk1.8                 ########################################### [100%]
Unpacking JAR files...
	tools.jar...
	plugin.jar...
	javaws.jar...
	deploy.jar...
	rt.jar...
	jsse.jar...
	charsets.jar...
	localedata.jar...
[root@CentOSX ~]# vi .bashrc 
JAVA_HOME=/usr/java/latest
CLASSPATH=.
PATH=$PATH:$JAVA_HOME/bin
export JAVA_HOME
export CLASSPATH
export PATH
[root@CentOSX ~]# vi .bashrc 

- 配置主机名为CentOS

[root@CentOS ~]# vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=CentOS[A,B,C]

- 配置主机名和IP映射

[root@CentOS ~]# vi /etc/hosts      
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
192.168.52.130 CentOSA
192.168.52.131 CentOSB
192.168.52.132 CentOSC

- 关闭防火墙

[root@CentOSX ~]# service iptables stop
iptables: Setting chains to policy ACCEPT: filter          [  OK  ]
iptables: Flushing firewall rules:                         [  OK  ]
iptables: Unloading modules:                               [  OK  ]
[root@CentOSX ~]# chkconfig iptables off
[root@CentOSX ~]# chkconfig --list | grep iptables
iptables       	0:关闭	1:关闭	2:关闭	3:关闭	4:关闭	5:关闭	6:关闭

- 安装配置Zookeeper

[root@CentOSX ~]# tar -zxf zookeeper-3.4.6.tar.gz -C /usr/
[root@CentOSX ~]# cd /usr/zookeeper-3.4.6/
[root@CentOSX zookeeper-3.4.6]# cp conf/zoo_sample.cfg conf/zoo.cfg
[root@CentOSX zookeeper-3.4.6]# vi conf/zoo.cfg 
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/root/zkdata
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
server.1=CentOSA::2888:3888
server.2=CentOSA::2888:3888
server.3=CentOSA::2888:3888
[root@CentOSX ~]# mkdir /root/zkdata
[root@CentOSA ~]# echo 1 > /root/zkdata/myid
[root@CentOSB ~]# echo 2 > /root/zkdata/myid
[root@CentOSC ~]# echo 3 > /root/zkdata/myid

- 启动zookeeper服务

[root@CentOSX zookeeper-3.4.6]# ./bin/zkServer.sh 
JMX enabled by default
Using config: /usr/zookeeper-3.4.6/bin/../conf/zoo.cfg
Usage: ./bin/zkServer.sh {start|start-foreground|stop|restart|status|upgrade|print-cmd}
[root@CentOS zookeeper-3.4.6]# ./bin/zkServer.sh start zoo.cfg
JMX enabled by default
Using config: /usr/zookeeper-3.4.6/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@CentOS zookeeper-3.4.6]# 
[root@CentOS zookeeper-3.4.6]# ./bin/zkServer.sh status zoo.cfg
JMX enabled by default
Using config: /usr/zookeeper-3.4.6/bin/../conf/zoo.cfg
Mode: leader|follower
[root@CentOSX zookeeper-3.4.6]# jps
1778 Jps
1733 QuorumPeerMain

- 安装配置Kafka集群

[root@CentOSX ~]# tar -zxf kafka_2.11-2.2.0.tgz -C /usr/
[root@CentOSX ~]# cd /usr/kafka_2.11-2.2.0/
[root@CentOSX kafka_2.11-2.2.0]# vi config/server.properties 
############################# Server Basics #############################

# The id of the broker. This must be set to a unique integer for each broker.
broker.id=[0,1,2]

############################# Socket Server Settings #############################

listeners=PLAINTEXT://CentOS[A,B,C]:9092

############################# Log Basics #############################

log.dirs=/usr/kafka-logs

############################# Log Retention Policy #############################

log.retention.hours=168

############################# Zookeeper #############################

zookeeper.connect=CentOSA:2181,CentOSB:2181,CentOSC:2181

[root@CentOSX kafka_2.11-2.2.0]#./bin/kafka-server-start.sh -daemon config/server.properties 

./bin/kafka-topics.sh --bootstrap-server Centos:9092 \ 

Topic管理

创建
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-topics.sh 
                    --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                    --create 
                    --topic topic02 
                    --partitions 3 
                    --replication-factor 3
列表
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-topics.sh 
                    --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                    --list
topic01
topic02
详情
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-topics.sh 
                    --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                    --describe 
                    --topic topic01
Topic:topic01	PartitionCount:3	ReplicationFactor:3	Configs:segment.bytes=1073741824
	Topic: topic01	Partition: 0	Leader: 0	Replicas: 0,2,3	Isr: 0,2,3
	Topic: topic01	Partition: 1	Leader: 2	Replicas: 2,3,0	Isr: 2,3,0
	Topic: topic01	Partition: 2	Leader: 0	Replicas: 3,0,2	Isr: 0,2,3
修改
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-topics.sh 
                    --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                    --create 
                    --topic topic03 
                    --partitions 1 
                    --replication-factor 1
                    
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-topics.sh 
                    --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                    --alter 
                    --topic topic03 
                    --partitions 2
删除
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-topics.sh 
                    --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                    --delete 
                    --topic topic03
订阅
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-console-consumer.sh 
                  --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                  --topic topic01 
                  --group g1 
                  --property print.key=true 
                  --property print.value=true 
                  --property key.separator=,
消费组
    [root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-consumer-groups.sh 
                  --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                  --list
                  g1
                  
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-consumer-groups.sh 
                  --bootstrap-server CentOSA:9092,CentOSB:9092,CentOSC:9092 
                  --describe 
                  --group g1

TOPIC PARTITION CURRENT-OFFSET LOG-END-OFFSET LAG CONSUMER-ID    HOST            CLIENT-ID
topic01 1       0              0               0  consumer-1-** /192.168.52.130 consumer-1
topic01 0       0              0               0  consumer-1-** /192.168.52.130 consumer-1
topic01 2       1              1               0  consumer-1-** /192.168.52.130 consumer-1
生产
[root@CentOSA kafka_2.11-2.2.0]# ./bin/kafka-console-producer.sh 
                  --broker-list CentOSA:9092,CentOSB:9092,CentOSC:9092 
                  --topic topic01
  • 0
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值