1.搭建kafka环境(本地/容器)
推荐容器环境
docker-compose-kafka.yml
version: '3.1'
services:
zookeeper:
image: wurstmeister/zookeeper
restart: always
kafka:
image: wurstmeister/kafka
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_HOST_NAME: localhost
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
docker-compose -f docker-compose-kafka.yml up -d 一键部署
如果要清除,用 docker-compose down --remove-orphans
2.springboot建立kafka-stream
pom.xml依赖
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<kafka.version>3.1.0</kafka.version>
</properties>
<!-- Apache Kafka dependencies -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
<version>${kafka.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
</dependency>
代码
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.Materialized;
import org.apache.kafka.streams.kstream.Produced;
import org.apache.kafka.streams.state.KeyValueStore;
import java.util.Arrays;
import java.util.Locale;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
public class WordCount {
public static void main(String[] args) throws Exception {
/**
* 1.创建一个java.util.Properties映射来指定在StreamsConfig配置值。
* (1)BOOTSTRAP_SERVERS_CONFIG,它指定了一个主机/端口对,表示Kafka地址;
* (2)APPLICATION_ID_CONFIG,它提供了Streams应用程序的唯一标识符;
* (3)其他配置,例如,记录键值对的默认序列化和反序列化库
*/
Properties props = new Properties();
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount");
props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
/**
* 2.定义Streams应用程序的计算逻辑。
* (1)定义为连接处理器节点的拓扑;
*/
final StreamsBuilder builder = new StreamsBuilder();
KStream<String, String> source = builder.stream("streams-plaintext-input");
source.flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+")))
.groupBy((key, value) -> value)
.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("counts-store"))
.toStream()
.to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long()));
final Topology topology = builder.build();
/**
* (2)将拓扑图放入KafkaStreams流client中
*/
final KafkaStreams streams = new KafkaStreams(topology, props);
/**
* 3.启动stream流,让其一直运行
* 通过调用它的start()函数,我们可以触发这个客户机的执行。在此客户机上调用close()之前,执行不会停止。
* 例如,我们可以添加一个带有倒计时锁闩的关机钩子来捕获用户中断,并在终止程序时关闭客户端:
*/
final CountDownLatch latch = new CountDownLatch(1);
// attach shutdown handler to catch control-c
Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") {
@Override
public void run() {
streams.close();
latch.countDown();
}
});
try {
streams.start();
latch.await();
} catch (Throwable e) {
System.exit(1);
}
System.exit(0);
}
}
3.调测
(1)创建topic主题
docker exec -it docker-kafka-1 sh
cd /opt/kafka_2.13-2.8.1
kafka-topics.sh --create \
--bootstrap-server localhost:9092 \
--replication-factor 1 \
--partitions 1 \
--topic streams-plaintext-input
Created topic "streams-plaintext-input"
kafka-topics.sh --create \
--bootstrap-server localhost:9092 \
--replication-factor 1 \
--partitions 1 \
--topic streams-wordcount-output \
--config cleanup.policy=compact
Created topic "streams-wordcount-output"
查看主题
kafka-topics.sh --bootstrap-server localhost:9092 --describe
(2)新建producer
kafka-console-producer.sh --bootstrap-server localhost:9092 --topic streams-plaintext-input
(3)新建consumer
kafka-console-consumer.sh --bootstrap-server localhost:9092 \
--topic streams-wordcount-output \
--from-beginning \
--formatter kafka.tools.DefaultMessageFormatter \
--property print.key=true \
--property print.value=true \
--property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer \
--property value.deserializer=org.apache.kafka.common.serialization.LongDeserializer
(4)启动stream
在springboot中启动
(5)生产者发送数据测试
完成!