SpringBoot集成kafka
尝试了三种方式来发送和接收消息
依赖文件
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka-test</artifactId>
<scope>test</scope>
</dependency>
普通Java的方法
根据上次学的kafka生产者Java客户端和消费者Java客户端编写一个简单的发送和接收消息的demo
private Logger logger = LoggerFactory.getLogger(getClass());
private String group = "group01";
private String topic1 = "topic1";
@Test
public void testAutoCommit() throws Exception{
logger.info("start auto");
//启动消费者
ContainerProperties containerProps = new ContainerProperties("topic1","topic2");
final CountDownLatch latch = new CountDownLatch(4);
containerProps.setMessageListener(new MessageListener<Integer,String>() {
@Override
public void onMessage(ConsumerRecord<Integer, String> integerStringConsumerRecord) {
logger.info("received:"+integerStringConsumerRecord);
latch.countDown();
}
});
KafkaMessageListenerContainer<Integer, String> container = createContainer(containerProps);
container.setBeanName("testauto");
container.start();
Thread.sleep(1000);
//启动生产者
KafkaTemplate<Integer,String> template = createTemplate();
template.setDefaultTopic(topic1);
template.sendDefault(0,"foo");
template.sendDefault(2,"bar");
template.sendDefault(0,"baz");
template.sendDefault(2,"qux");
template.flush();
assertTrue(latch.await(60, TimeUnit.SECONDS));
container.stop(); // 关闭消费者
logger.info("Stop auto");
}
private KafkaTemplate<Integer, String> createTemplate() {
Map<String, Object> senderProps = senderProps();
ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<Integer, String>(senderProps);
KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
return template;
}
private Map<String, Object> senderProps() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(ProducerConfig.RETRIES_CONFIG, 0);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
return props;
}
private KafkaMessageListenerContainer<Integer, String> createContainer(ContainerProperties containerProps) {
Map<String, Object> props = consumerProps();
DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props);
KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf, containerProps);
return container;
}
private Map<String, Object> consumerProps() {
Map<String, Object> props = new HashMap<>();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG, group);
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
return props;
}
运行效果图
Java配置的方法
package com.xfgg.demo.config;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.IntegerDeserializer;
import org.apache.kafka.common.serialization.IntegerSerializer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.*;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
@Configuration
@EnableKafka
public class Config { @Autowired
private Listener listener;
@Autowired @Autowired
private Listener listener;
@Autowired
private KafkaTemplate<Integer, String> template;
@Test
public void testSimple() throws Exception{
template.send("annotated1",0,"foo");
template.flush();
assertTrue(this.listener.latch1.await(10, TimeUnit.SECONDS));
}
private KafkaTemplate<Integer, String> template;
@Test
public void testSimple() throws Exception{
template.send("annotated1",0,"foo");
template.flush();
assertTrue(this.listener.latch1.await(10, TimeUnit.SECONDS));
}
private String group = "group01";
@Bean
ConcurrentKafkaListenerContainerFactory<Integer, String>
kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<Integer, String> factory =
new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
return factory;
}
@Bean
public ConsumerFactory<Integer, String> consumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
@Bean
public Map<String, Object> consumerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG, group);
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
return props;
}
@Autowired
private Listener listener;
@Autowired
private KafkaTemplate<Integer, String> template;
@Test
public void testSimple() throws Exception{
template.send("annotated1",0,"foo");
template.flush();
assertTrue(this.listener.latch1.await(10, TimeUnit.SECONDS));
}
@Bean
public Listener listener() {
return new Listener();
}
@Bean
public ProducerFactory<Integer, String> producerFactory() {
return new DefaultKafkaProducerFactory<>(producerConfigs());
}
@Bean
public Map<String, Object> producerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(ProducerConfig.RETRIES_CONFIG, 0);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
return props;
}
@Bean
public KafkaTemplate<Integer, String> kafkaTemplate() {
return new KafkaTemplate<Integer, String>(producerFactory());
}
}
class Listener {
private final CountDownLatch latch1 = new CountDownLatch(1);
@KafkaListener(id = "foo", topics = "annotated1")
public void listen1(String foo) {
this.latch1.countDown();
}
}
@Autowired
private Listener listener;
@Autowired
private KafkaTemplate<Integer, String> template;
@Test
public void testSimple() throws Exception{
template.send("annotated1",0,"foo");
template.flush();
assertTrue(this.listener.latch1.await(10, TimeUnit.SECONDS));
}
运行效果图
SpringBoot方法
@SpringBootApplication
public class DemoApplication implements CommandLineRunner{
public static Logger logger = LoggerFactory.getLogger(DemoApplication.class);
public static void main(String[] args) {
SpringApplication.run(DemoApplication.class, args).close();
}
@Autowired
private KafkaTemplate<Integer, String> template;
private final CountDownLatch latch = new CountDownLatch(3);
@Override
public void run(String... args) throws Exception {
this.template.send("myTopic", "foo1");
this.template.send("myTopic", "foo2");
this.template.send("myTopic", "foo3");
latch.await(60, TimeUnit.SECONDS);
logger.info("All received");
}
@KafkaListener(topics = "myTopic")
public void listen(ConsumerRecord<?, ?> cr) throws Exception {
logger.info(cr.toString());
latch.countDown();
}
}
发现会报错,提示导入出错
修改成Resource仍然报错
不知道为什么按照提示将autowired那一块重新写了一遍就可以了
估计写错了,懵比
运行效果图
KafkaStream客户端
kafka Streams从一个或多个输入topic进行连续的计算并输出到0或者多个外部topic中,
可以通过TopologyBuilder类定义一个计算逻辑处理器DAG拓扑,或者也可以通过提供的高级别KStream DSL来定义转换的KStreamBuilder
KafkaStreams类管理Kafka Streams实例的生命周期。一个stream实例可以在配置文件中为处理器指定一个或多个Thread。
KafkaStreams实例可以作为单个streams处理客户端(也可能是分布式的),与其他的相同应用ID的实例进行协调(无论是否在同一个进程中,在同一台机器的其他进程中,或远程机器上)。这些实例将根据输入topic分区的基础上来划分工作,以便所有的分区都被消费掉。如果实例添加或失败,所有实例将重新平衡它们之间的分区分配,以保证负载平衡。
在内部,KafkaStreams实例包含一个正常的KafkaProducer和KafkaConsumer实例,用于读取和写入
Map<String, Object> props = new HashMap<>();
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "my-stream-processing-application");
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
StreamsConfig config = new StreamsConfig(props);
KStreamBuilder builder = new KStreamBuilder();
builder.stream("my-input-topic").mapValues(value -> value.length().toString()).to("my-output-topic");
KafkaStreams streams = new KafkaStreams(builder, config);
streams.start();
kafka管理Java客户端
Kafka的管理Java客户端,支持管理和检查topic、broker、配置和ACL
创建topic
//创建topic
private void createTopics(String bootstrapServers){
Properties properties = new Properties();
properties.put("bootstrap.servers", bootstrapServers);
properties.put("connections.max.idle.ms", 10000);
properties.put("request.timeout.ms", 5000);
try (AdminClient client=AdminClient.create(properties)){
CreateTopicsResult result = client.createTopics(Arrays.asList(
new NewTopic("topic1",1,(short) 1),
new NewTopic("topic2",1,(short) 1),
new NewTopic("topic3",1,(short) 1)
));
try {
result.all().get();
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
}
topic列表
//topic列表
private void listTopics(String bootstrapServers){
Properties properties = new Properties();
properties.put("bootstrap.servers", bootstrapServers);
properties.put("connections.max.idle.ms", 10000);
properties.put("request.timeout.ms", 5000);
try (AdminClient client = AdminClient.create(properties)) {
ListTopicsResult result = client.listTopics();
try {
result.listings().get().forEach(topic -> {
System.out.println(topic);
});
} catch (InterruptedException | ExecutionException e) {
throw new IllegalStateException(e);
}
}
增加分区
作者:半兽人
链接:https://www.orchome.com/9994
来源:OrcHome
著作权归作者所有。商业转载请联系作者获得授权,非商业转载请注明出处。
Properties properties = new Properties();
properties.put("bootstrap.servers", bootstrapServers);
properties.put("connections.max.idle.ms", 10000);
properties.put("request.timeout.ms", 5000);
try (AdminClient client = AdminClient.create(properties)) {
Map newPartitions = new HashMap<>();
// 增加到2个
newPartitions.put("topic1", NewPartitions.increaseTo(2));
CreatePartitionsResult rs = client.createPartitions(newPartitions);
try {
rs.all().get();
} catch (InterruptedException | ExecutionException e) {
throw new IllegalStateException(e);
}
}
kafka的配置
可以配置的属性有点多,就不一一阐述了,简单的看了一遍,打算通过实践来慢慢熟悉
主要有Broker配置,Topic配置,Producer配置,Consumer配置,Stream配置,Connect配置,AdminClient配置
详情可见:https://www.orchome.com/472
Spring Cloud整合kafka
应用程序通过Spring Cloud Stream注入到输入和输出通道与外界进行通信,根据此规则很容易的实现消息传递,订阅消息与消息中转,并且当需要切换消息中间件时,几乎不需要修改代码,只需要变更配置
在用例图中 Inputs代表了应用程序监听消息 、outputs代表发送消息、binder的话大家可以理解为将应用程序与消息中间件隔离的抽象,类似于三层架构下利用dao屏蔽service与数据库的实现的原理。
在配置中心中配置kafka
依赖文件
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-config</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
<version>2.0.1.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
</dependency>
配置文件
application.yml
server:
port: 9900
eureka:
client:
service-url:
defaultZone: http://${eureka.instance.hostname}:8761/eureka/
instance:
prefer-ip-address: true
hostname: ${spring.cloud.client.ip-address}
instance-id: ${spring.cloud.client.ip-address}:${spring.application.name}:${server.port}
spring:
application:
name: testkafka
cloud:
stream:
kafka:
binder:
brokers: localhost:9092
bindings:
#TestStream中INPUT
test-in:
destination: testkafka
contentType: application/json
#TestStream中OUTPUT
test-out:
destination: testkafka
contentType: application/json
bootstrap.yml
spring:
cloud:
config:
uri: http://${host:localhost}:9020
name: config
profile: ${active:dev}
server:
servlet:
encoding:
charset: UTF-8
enabled: true
force: true
启动类
package com.xfgg.demo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
@SpringBootApplication
@EnableEurekaClient
public class DemoApplication {
public static void main(String[] args) {
SpringApplication.run(DemoApplication.class, args);
}
}
TestStream接口
package com.xfgg.demo.stream;
import org.springframework.cloud.stream.annotation.Input;
import org.springframework.cloud.stream.annotation.Output;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.SubscribableChannel;
public interface TestStream {
String INPUT = "test-in";
String OUTPUT ="test-out";
@Input(INPUT)
SubscribableChannel testIn();
@Output(OUTPUT)
MessageChannel testOut();
}
StreamReceiver实现类
package com.xfgg.demo.stream;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.annotation.StreamListener;
import org.springframework.stereotype.Component;
@Component
@EnableBinding(value = {TestStream.class})
public class StreamReceiver {
@StreamListener(TestStream.INPUT)
public void receive(String message){
System.out.println("StreamReceiver:"+message);
}
}
SendController控制层
package com.xfgg.demo.controller;
import com.xfgg.demo.stream.TestStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.support.MessageBuilder;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("api/")
public class SendController {
@Autowired
private TestStream testStream;
@GetMapping("send")
public void send(){
System.out.println("send");
testStream.testOut().send(MessageBuilder.withPayload("Hello World...").build());
System.out.println("ok");
}
}
运行注册中心,配置中心,启动kafka服务
效果图
在服务中配置kafka
很简单,在原来的基础上,把StreamReceiver和TestStream拷贝到服务C中
添加依赖
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
<version>2.0.1.RELEASE</version>
</dependency>
配置文件
spring.cloud.stream.kafka.binder.brokers=localhost:9092
spring.cloud.stream.bindings.test-in.destination=testkafka
streamReceiver
package com.xfgg.demo.kafka;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.annotation.StreamListener;
import org.springframework.stereotype.Component;
@Component
@EnableBinding(value = {TestStream.class})
public class StreamReceiver {
@StreamListener(TestStream.INPUT)
public void receive(String message){
System.out.println("StreamReceiver:"+message);
}
}
TestStream
package com.xfgg.demo.kafka;
import org.springframework.cloud.stream.annotation.Input;
import org.springframework.cloud.stream.annotation.Output;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.SubscribableChannel;
public interface TestStream {
String INPUT = "test-in";
String OUTPUT = "test-out";
@Input(INPUT)
SubscribableChannel testIn();
@Output(OUTPUT)
MessageChannel testOut();
}
运行是成功运行了,但是却显示没有消息经过INPUT那里,访问api/send,服务C控制台也并没有输出数据
效果图
个人感觉是配置文件出现了问题,可以运行但是没有数据流,可以从输出的send ok知道消息是发送出去了,估计是接收消息出现了问题
查看了网上大佬的配置文件
配置文件
spring:
application:
name: shop-server
cloud:
stream:
bindings:
#配置自己定义的通道与哪个中间件交互
shop_input: #ShopChannel里Input和Output的值
destination: testkafka #目标主题
shop_output:
destination: testkafka
default-binder: kafka #默认的binder是kafka
kafka:
bootstrap-servers: localhost:9092 #kafka服务地址
consumer:
group-id: consumer1
producer:
key-serializer: org.apache.kafka.common.serialization.ByteArraySerializer
value-serializer: org.apache.kafka.common.serialization.ByteArraySerializer
client-id: producer1
具体不行的原因还尚未知晓,打算明天重构一下