Kafka Streams开发单词计数应用

先添加依赖

<dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-streams</artifactId>
            <version>2.0.0</version>
        </dependency>

代码如下：

package KafkaStreams;

import org.apache.kafka.streams.processor.Processor;
import org.apache.kafka.streams.processor.ProcessorContext;
import java.util.HashMap;

public class LogProcessor implements Processor<byte[], byte[]> {
    private ProcessorContext processorContext;
    @Override
    public void init(ProcessorContext processorContext) {
        this.processorContext = processorContext;
    }

    @Override
    public void process(byte[] key, byte[] value) {
        String inputOri = new String(value);
            HashMap<String, Integer>map = new HashMap<String, Integer>();
            int times = 1;
            if (inputOri.contains(" ")){
                //截取字段
                String[] words = inputOri.split(" ");
                for (String word:words){
                    if (map.containsKey(word)){
                        map.put(word, map.get(word)+1);
                    }else {
                        map.put(word, times);
                    }
                }
            }
            inputOri = map.toString();
            processorContext.forward(key, inputOri.getBytes());
    }

    @Override
    public void close() {

    }
}

package KafkaStreams;

import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.processor.Processor;
import org.apache.kafka.streams.processor.ProcessorSupplier;
import org.apache.kafka.common.utils.LogContext;
import java.util.Properties;

public class App {
    public static void main(String[] args){
        //声明来源主题
        String fromTopic = "testStreams1";
        //声明目标主题
        String toTopic = "testStreams2";
        //设置KafkaStreams参数信息
        Properties props = new Properties();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "logProcessor");
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.110.12:9092");
        //实例化StreamsConfig对象
        StreamsConfig config = new StreamsConfig(props);
        //创建拓扑结构
        Topology topology = new Topology();
        //添加处理节点，为源处理节点指定名称和它订阅的主题
        topology.addSource("SOURCE", fromTopic)
                //添加自定义处理节点，指定处理器类和上一节点的名称
                .addProcessor("PROCESSOR", new ProcessorSupplier() {
                    @Override
                    public Processor get() {
                        return new LogProcessor();
                    }
                },"SOURCE")
                //添加目标处理节点，需要指定目标处理节点和上一节点的名称
                .addSink("SINK", toTopic, "PROCESSOR");
        //实例化KafkaStreams对象
        KafkaStreams streams = new KafkaStreams(topology, config);
        streams.start();
    }
}

代码调试结束后，开启集群zookeeper和kafka
在Kafka主节点创建testStreams1和testStreams2主题，命令如下：

bin/kafka-topics.sh --zookeeper master:2181,slave1:2181,slave2:2181 \
--create --replication-factor 2 --partitions 3 --topic testStreams1

创建好后分别在slave1和slave2启动生产者和消费者
启动生产者

[root@slave1 kafka]# bin/kafka-console-producer.sh --broker-list slave1:9092 --topic testStreams1

启动消费者

[root@slave2 kafka]# bin/kafka-console-consumer.sh --bootstrap-server slave1:9092 --topic testStreams2

然后运行主程序代码。
在生产者节点输入hello itcast hello spark
会在消费者节点查看到{hello=2，itcast=1，spark=1}