先添加依赖
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
<version>2.0.0</version>
</dependency>
代码如下:
package KafkaStreams;
import org.apache.kafka.streams.processor.Processor;
import org.apache.kafka.streams.processor.ProcessorContext;
import java.util.HashMap;
public class LogProcessor implements Processor<byte[], byte[]> {
private ProcessorContext processorContext;
@Override
public void init(ProcessorContext processorContext) {
this.processorContext = processorContext;
}
@Override
public void process(byte[] key, byte[] value) {
String inputOri = new String(value);
HashMap<String, Integer>map = new HashMap<String, Integer>();
int times = 1;
if (inputOri.contains(" ")){
//截取字段
String[] words = inputOri.split(" ");
for (String word:words){
if (map.containsKey(word)){
map.put(word, map.get(word)+1);
}else {
map.put(word, times);
}
}
}
inputOri = map.toString();
processorContext.forward(key, inputOri.getBytes());
}
@Override
public void close() {
}
}
package KafkaStreams;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.processor.Processor;
import org.apache.kafka.streams.processor.ProcessorSupplier;
import org.apache.kafka.common.utils.LogContext;
import java.util.Properties;
public class App {
public static void main(String[] args){
//声明来源主题
String fromTopic = "testStreams1";
//声明目标主题
String toTopic = "testStreams2";
//设置KafkaStreams参数信息
Properties props = new Properties();
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "logProcessor");
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.110.12:9092");
//实例化StreamsConfig对象
StreamsConfig config = new StreamsConfig(props);
//创建拓扑结构
Topology topology = new Topology();
//添加处理节点,为源处理节点指定名称和它订阅的主题
topology.addSource("SOURCE", fromTopic)
//添加自定义处理节点,指定处理器类和上一节点的名称
.addProcessor("PROCESSOR", new ProcessorSupplier() {
@Override
public Processor get() {
return new LogProcessor();
}
},"SOURCE")
//添加目标处理节点,需要指定目标处理节点和上一节点的名称
.addSink("SINK", toTopic, "PROCESSOR");
//实例化KafkaStreams对象
KafkaStreams streams = new KafkaStreams(topology, config);
streams.start();
}
}
代码调试结束后,开启集群zookeeper和kafka
在Kafka主节点创建testStreams1和testStreams2主题,命令如下:
bin/kafka-topics.sh --zookeeper master:2181,slave1:2181,slave2:2181 \
--create --replication-factor 2 --partitions 3 --topic testStreams1
创建好后分别在slave1和slave2启动生产者和消费者
启动生产者
[root@slave1 kafka]# bin/kafka-console-producer.sh --broker-list slave1:9092 --topic testStreams1
启动消费者
[root@slave2 kafka]# bin/kafka-console-consumer.sh --bootstrap-server slave1:9092 --topic testStreams2
然后运行主程序代码。
在生产者节点输入hello itcast hello spark
会在消费者节点查看到{hello=2,itcast=1,spark=1}