Kafka Stream窗口操作
Kafka Stream窗口
Hopping Time Window
- 编写Java类
package nj.zb.kb09.kafka;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.protocol.types.Field;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.TimeWindows;
import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
public class windowStream {
public static void main(String[] args) {
Properties prop = new Properties();
prop.put(StreamsConfig.APPLICATION_ID_CONFIG, "windowStream1");
prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.136.100:9092");
prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 3000);
prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
StreamsBuilder builder = new StreamsBuilder();
KStream<Object, Object> source = builder.stream("windowdemo");
source.flatMapValues(value-> Arrays.asList(value.toString().split("\\s+")))
.map((x,y)->
{
return new KeyValue<String,String>(y,"1");
}).groupByKey()
//Hopping Time Window
.windowedBy(TimeWindows.of(Duration.ofSeconds(5).toMillis()).advanceBy(Duration.ofSeconds(2).toMillis()))
.count().toStream().foreach((x,y)->{
System.out.println("x:"+x+" y:"+y);
});
final Topology topo = builder.build();
final KafkaStreams streams = new KafkaStreams(topo,prop);
final CountDownLatch latch = new CountDownLatch(1);
Runtime.getRuntime().addShutdownHook(new Thread("stream"){
@Override
public void run() {
streams.close();
latch.countDown();
}
});
streams.start();
try {
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
System.exit(0);
}
}
- 运行Java进程
- 根据编写的Java类创建topic
[root@hadoop100 ~]# kafka-topics.sh --zookeeper 192.168.136.100:2181 --create --topic windowdemo --partitions 1 --replication-factor 1
- 在windowdemo队列创建生产者信息
[root@hadoop100 opt]# kafka-console-producer.sh --topic windowdemo --broker-list 192.168.136.100:9092
- 在windowdemo生产消息
hello java
hello world
- 对应的也在windowdemo消费到了消息
Tumbling Time Window
- 编写Java类
package nj.zb.kb09.kafka;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.protocol.types.Field;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.TimeWindows;
import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
public class windowStream {
public static void main(String[] args) {
Properties prop = new Properties();
prop.put(StreamsConfig.APPLICATION_ID_CONFIG, "windowStream2");
prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.136.100:9092");
prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 3000);
prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
StreamsBuilder builder = new StreamsBuilder();
KStream<Object, Object> source = builder.stream("windowdemo");
source.flatMapValues(value-> Arrays.asList(value.toString().split("\\s+")))
.map((x,y)->
{
return new KeyValue<String,String>(y,"1");
}).groupByKey()
//Tumbling Time Window
.windowedBy(TimeWindows.of(Duration.ofSeconds(5).toMillis()))
.count().toStream().foreach((x,y)->{
System.out.println("x:"+x+" y:"+y);
});
final Topology topo = builder.build();
final KafkaStreams streams = new KafkaStreams(topo,prop);
final CountDownLatch latch = new CountDownLatch(1);
Runtime.getRuntime().addShutdownHook(new Thread("stream"){
@Override
public void run() {
streams.close();
latch.countDown();
}
});
streams.start();
try {
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
System.exit(0);
}
}
- 运行Java进程
- 在windowdemo生产消息
hello java
hello world
hello scala
hello scala
hello spark
hello spark
hello spark
hello spark
hello spark
hello spark
hello spark
hello spark
- 对应的在windowdemo消费到了消息
Session Window
- 编写Java类
package nj.zb.kb09.kafka;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.protocol.types.Field;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.TimeWindows;
import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
public class windowStream {
public static void main(String[] args) {
Properties prop = new Properties();
prop.put(StreamsConfig.APPLICATION_ID_CONFIG, "windowStream3");
prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.136.100:9092");
prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 3000);
prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
StreamsBuilder builder = new StreamsBuilder();
KStream<Object, Object> source = builder.stream("windowdemo");
source.flatMapValues(value-> Arrays.asList(value.toString().split("\\s+")))
.map((x,y)->
{
return new KeyValue<String,String>(y,"1");
}).groupByKey()
.windowedBy(SessionWindows.with(Duration.ofSeconds(15).toMillis()))
.count().toStream().foreach((x,y)->{
System.out.println("x:"+x+" y:"+y);
});
final Topology topo = builder.build();
final KafkaStreams streams = new KafkaStreams(topo,prop);
final CountDownLatch latch = new CountDownLatch(1);
Runtime.getRuntime().addShutdownHook(new Thread("stream"){
@Override
public void run() {
streams.close();
latch.countDown();
}
});
streams.start();
try {
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
System.exit(0);
}
}
- 运行Java进程
- 在windowdemo生产消息
hello java
- 对应的在windowdemo消费到了消息
注意:每次编写Java类时,application_id_config的名字都要不一样。