上一篇文章我们已经成功安装了kafka,本文讲解部署kafka集群,并使用springboot整合测试。
设置多 broker 集群
由于只有一台虚拟机,于是通过多个配置文件模拟多台broker
首先为每个broker创建一个配置文件:
cp config/server.properties config/server-1.propertie
cp config/server.properties config/server-2.properties
现在编辑这些新建的文件,设置以下属性
config/server.properties:
broker.id=0
#此处填写你的服务器ip
listeners=PLAINTEXT://192.168.130.128:9092
#选择你的logs存放目录
log.dirs=/usr/kafka2.12/kafka-logs
delete.topic.enable=true
#zookeeper集群信息
zookeeper.connect=192.168.130.128:2181,192.168.130.128:2182,192.168.130.128:2183
config/server-1.properties:
broker.id=1
#此处填写你的服务器ip
listeners=PLAINTEXT://192.168.130.128:9093
#选择你的logs存放目录
log.dirs=/usr/kafka2.12/kafka-logs-1
delete.topic.enable=true
#zookeeper集群信息
zookeeper.connect=192.168.130.128:2181,192.168.130.128:2182,192.168.130.128:2183
config/server-2.properties:
broker.id=0
#此处填写你的服务器ip
listeners=PLAINTEXT://192.168.130.128:9094
#选择你的logs存放目录
log.dirs=/usr/kafka2.12/kafka-logs-2
delete.topic.enable=true
#zookeeper集群信息
zookeeper.connect=192.168.130.128:2181,192.168.130.128:2182,192.168.130.128:2183
broker.id属性是集群中每个节点的名称,这一名称是唯一且永久的。
启动三个的节点
bin/kafka-server-start.sh config/server.properties &
bin/kafka-server-start.sh config/server-1.properties &
bin/kafka-server-start.sh config/server-2.properties &
现在创建一个副本为3的新 topic:
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic new_topic
当其中一台broker挂掉时,再次启动执行此命令重置优先副本
bin/kafka-preferred-replica-election.sh --zookeeper localhost:2181
springboot整合kafka
导入依赖
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.3.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
application.yml
spring:
application:
name: spring-boot-kafka
# kafka
kafka:
# 指定 kafka集群地址
bootstrap-servers:
- 192.168.130.128:9092
- 192.168.130.128:9093
- 192.168.130.128:9094
# 指定listener 容器中的线程数,用于提高并发量
listener:
concurrency: 3
# 生产者的配置
producer:
# 每次批量发送消息的数量
batch-size: 1000
retries: 0
buffer-memory: 33554432
# key,value序列化方式
key-serializer: org.apache.kafka.common.serialization.StringSerializer
value-serializer: org.apache.kafka.common.serialization.StringSerializer
# 消费者的配置
consumer:
# 指定默认消费者group id
group-id: test-group2
auto-offset-reset: latest
# 是否开启自动提交
enable-auto-commit: true
# 自动提交的时间间隔
auto-commit-interval: 1000
# key,value的解码方式
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
# 指定默认topic id
template:
default-topic: new_topic
引导类
package com.sunyuqi.springboot;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class KafkaDemoApplication {
public static void main(String[] args) {
SpringApplication.run(KafkaDemoApplication.class, args);
}
}
生产者
package com.sunyuqi.springboot.producer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.stereotype.Component;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;
@Component
public class Producer {
private static final Logger log = LoggerFactory.getLogger(Producer.class);
@Autowired
private KafkaTemplate<String, String> kafkaTemplate;
public void sendMessage(String topic, String data) {
log.info("kafka sendMessage start");
ListenableFuture<SendResult<String, String>> future = kafkaTemplate.send(topic, data);
future.addCallback(new ListenableFutureCallback<SendResult<String, String>>() {
@Override
public void onFailure(Throwable ex) {
log.error("kafka sendMessage error, ex = {}, topic = {}, data = {}", ex, topic, data);
}
@Override
public void onSuccess(SendResult<String, String> result) {
log.info("kafka sendMessage success topic = {}, data = {}",topic, data);
}
});
log.info("kafka sendMessage end");
}
}
消费者
package com.sunyuqi.springboot.consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
@Component
public class Consumer {
@KafkaListener(topics = {"new_topic"})
public void processMessage(ConsumerRecord<?, ?> record) {
System.out.printf("topic is %s, offset is %d, value is %s \n", record.topic(), record.offset(), record.value());
}
}
运行引导类,监听消息
测试类
package com.sunyuqi.springboot;
import com.sunyuqi.springboot.producer.Producer;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest
public class KafkaDemoApplicationTests {
@Autowired
private Producer producer;
@Test
public void contextLoads() {
producer.sendMessage("new_topic", "hello world");
}
}
运行测试类,发送消息
成功接收到消息