kafka-stream官方文档例子解析+springboot集成

1.搭建kafka环境（本地/容器）

推荐容器环境

docker-compose-kafka.yml

version: '3.1'

services:
  zookeeper:
    image: wurstmeister/zookeeper
    restart: always

  kafka:
    image:  wurstmeister/kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: localhost
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

docker-compose  -f docker-compose-kafka.yml up -d 一键部署

如果要清除，用 docker-compose down --remove-orphans

2.springboot建立kafka-stream

pom.xml依赖

<properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <kafka.version>3.1.0</kafka.version>
</properties>

<!-- Apache Kafka dependencies -->
<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-streams</artifactId>
    <version>${kafka.version}</version>
    <exclusions>
        <exclusion>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
        </exclusion>
    </exclusions>
</dependency>
<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-clients</artifactId>
    <version>${kafka.version}</version>
</dependency>
<dependency>
    <groupId>org.springframework.cloud</groupId>
    <artifactId>spring-cloud-starter-stream-kafka</artifactId>
</dependency>

代码

import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.Materialized;
import org.apache.kafka.streams.kstream.Produced;
import org.apache.kafka.streams.state.KeyValueStore;

import java.util.Arrays;
import java.util.Locale;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class WordCount {

    public static void main(String[] args) throws Exception {
        /**
         *  1.创建一个java.util.Properties映射来指定在StreamsConfig配置值。
         *  （1）BOOTSTRAP_SERVERS_CONFIG，它指定了一个主机/端口对，表示Kafka地址；
         *  （2）APPLICATION_ID_CONFIG，它提供了Streams应用程序的唯一标识符；
         *  （3）其他配置，例如，记录键值对的默认序列化和反序列化库
         */
        Properties props = new Properties();
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount");
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());

        /**
         * 2.定义Streams应用程序的计算逻辑。
         *   (1)定义为连接处理器节点的拓扑；
         */
        final StreamsBuilder builder = new StreamsBuilder();
        KStream<String, String> source = builder.stream("streams-plaintext-input");
        source.flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+")))
                .groupBy((key, value) -> value)
                .count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("counts-store"))
                .toStream()
                .to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long()));
        final Topology topology = builder.build();
        /**
         * （2）将拓扑图放入KafkaStreams流client中
         */
        final KafkaStreams streams = new KafkaStreams(topology, props);


        /**
         * 3.启动stream流，让其一直运行
         * 通过调用它的start()函数，我们可以触发这个客户机的执行。在此客户机上调用close()之前，执行不会停止。
         * 例如，我们可以添加一个带有倒计时锁闩的关机钩子来捕获用户中断，并在终止程序时关闭客户端:
         */
        final CountDownLatch latch = new CountDownLatch(1);
        // attach shutdown handler to catch control-c
        Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") {
            @Override
            public void run() {
                streams.close();
                latch.countDown();
            }
        });

        try {
            streams.start();
            latch.await();
        } catch (Throwable e) {
            System.exit(1);
        }
        System.exit(0);
    }
}

3.调测

（1）创建topic主题

docker exec -it docker-kafka-1 sh

cd /opt/kafka_2.13-2.8.1

kafka-topics.sh --create \
    --bootstrap-server localhost:9092 \
    --replication-factor 1 \
    --partitions 1 \
    --topic streams-plaintext-input
Created topic "streams-plaintext-input"

kafka-topics.sh --create \
    --bootstrap-server localhost:9092 \
    --replication-factor 1 \
    --partitions 1 \
    --topic streams-wordcount-output \
    --config cleanup.policy=compact
Created topic "streams-wordcount-output"

查看主题

kafka-topics.sh --bootstrap-server localhost:9092 --describe

（2）新建producer

 kafka-console-producer.sh --bootstrap-server localhost:9092 --topic streams-plaintext-input

（3）新建consumer

kafka-console-consumer.sh --bootstrap-server localhost:9092 \
    --topic streams-wordcount-output \
    --from-beginning \
    --formatter kafka.tools.DefaultMessageFormatter \
    --property print.key=true \
    --property print.value=true \
    --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer \
    --property value.deserializer=org.apache.kafka.common.serialization.LongDeserializer

（4）启动stream

在springboot中启动

（5）生产者发送数据测试

 

 完成！