kafka stream实现kafka数据流的word count示例。
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.kstream.*;
import java.util.Arrays;
import java.util.Properties;
/**
* @author feng.wei
*/
public class WordCountDemo {
public static Properties getProperties() {
Properties props = new Properties();
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-WordCountDemo");
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka.bootstrap.servers}");
props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, "${zk.servers}");
props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
return props;
}
public static void main(String[] args) {
// 窗口大小
Long windowSize = 1000 * 10L;
// 前后窗口的时间间隔
Long interval = windowSize;
KStreamBuilder builder = new KStreamBuilder();
// 指定topics,多个topic逗号分隔
KStream<String, String> source = builder.stream("test");
KTable<Windowed<String>, Long> countTable =
// 对每一行按照空格做切割
source.flatMapValues(new ValueMapper<String, Iterable<String>>() {
@Override
public Iterable<String> apply(String value) {
return Arrays.asList(value.split(" "));
}
})
// 对每个kv元素做变换
.map(new KeyValueMapper<String, String, KeyValue<String, String>>() {
@Override
public KeyValue<String, String> apply(String key, String value) {
return new KeyValue<>(value, value);
}
})
// 对每个时间窗口进行wordcount
.countByKey(TimeWindows.of("window-wordcount", windowSize).advanceBy(interval)); //
countTable.foreach(new ForeachAction<Windowed<String>, Long>() {
@Override
public void apply(Windowed<String> key, Long value) {
System.out.println("key:" + key.key() + ", value:" + value);
}
});
// 实例化KafkaStreams
KafkaStreams streams = new KafkaStreams(builder, getProperties());
streams.start();
//streams.close();
}
}