大数据——Kafka Stream窗口操作

Kafka Stream窗口

Hopping Time Window

  • 编写Java类
package nj.zb.kb09.kafka;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.protocol.types.Field;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.TimeWindows;

import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class windowStream {
	public static void main(String[] args) {
		Properties prop = new Properties();
		prop.put(StreamsConfig.APPLICATION_ID_CONFIG, "windowStream1");
		prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.136.100:9092");
		prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 3000);
		prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
		prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
		prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
		prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());

		StreamsBuilder builder = new StreamsBuilder();


		KStream<Object, Object> source = builder.stream("windowdemo");
		source.flatMapValues(value-> Arrays.asList(value.toString().split("\\s+")))
				.map((x,y)->
				{
					return new KeyValue<String,String>(y,"1");
				}).groupByKey()
				
				//Hopping Time Window
				.windowedBy(TimeWindows.of(Duration.ofSeconds(5).toMillis()).advanceBy(Duration.ofSeconds(2).toMillis()))
				.count().toStream().foreach((x,y)->{
			System.out.println("x:"+x+" y:"+y);
		});



		final Topology topo = builder.build();
		final KafkaStreams streams = new KafkaStreams(topo,prop);

		final CountDownLatch latch = new CountDownLatch(1);
		Runtime.getRuntime().addShutdownHook(new Thread("stream"){
			@Override
			public void run() {
				streams.close();
				latch.countDown();
			}
		});
		streams.start();
		try {
			latch.await();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}

		System.exit(0);
	}
}

  • 运行Java进程
  • 根据编写的Java类创建topic
[root@hadoop100 ~]# kafka-topics.sh --zookeeper 192.168.136.100:2181 --create --topic windowdemo --partitions 1 --replication-factor 1
  • 在windowdemo队列创建生产者信息
[root@hadoop100 opt]# kafka-console-producer.sh --topic windowdemo --broker-list 192.168.136.100:9092
  • 在windowdemo生产消息
hello java
hello world
  • 对应的也在windowdemo消费到了消息
    在这里插入图片描述

Tumbling Time Window

  • 编写Java类
package nj.zb.kb09.kafka;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.protocol.types.Field;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.TimeWindows;

import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class windowStream {
	public static void main(String[] args) {
		Properties prop = new Properties();
		prop.put(StreamsConfig.APPLICATION_ID_CONFIG, "windowStream2");
		prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.136.100:9092");
		prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 3000);
		prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
		prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
		prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
		prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());

		StreamsBuilder builder = new StreamsBuilder();


		KStream<Object, Object> source = builder.stream("windowdemo");
		source.flatMapValues(value-> Arrays.asList(value.toString().split("\\s+")))
				.map((x,y)->
				{
					return new KeyValue<String,String>(y,"1");
				}).groupByKey()
				//Tumbling Time Window
				.windowedBy(TimeWindows.of(Duration.ofSeconds(5).toMillis()))
				.count().toStream().foreach((x,y)->{
			System.out.println("x:"+x+" y:"+y);
		});



		final Topology topo = builder.build();
		final KafkaStreams streams = new KafkaStreams(topo,prop);

		final CountDownLatch latch = new CountDownLatch(1);
		Runtime.getRuntime().addShutdownHook(new Thread("stream"){
			@Override
			public void run() {
				streams.close();
				latch.countDown();
			}
		});
		streams.start();
		try {
			latch.await();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}

		System.exit(0);
	}
}
  • 运行Java进程
  • 在windowdemo生产消息
hello java
hello world
hello scala
hello scala
hello spark
hello spark
hello spark
hello spark
hello spark
hello spark
hello spark
hello spark
  • 对应的在windowdemo消费到了消息

在这里插入图片描述

Session Window

  • 编写Java类
package nj.zb.kb09.kafka;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.protocol.types.Field;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.TimeWindows;

import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class windowStream {
	public static void main(String[] args) {
		Properties prop = new Properties();
		prop.put(StreamsConfig.APPLICATION_ID_CONFIG, "windowStream3");
		prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.136.100:9092");
		prop.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 3000);
		prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
		prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
		prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
		prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());

		StreamsBuilder builder = new StreamsBuilder();


		KStream<Object, Object> source = builder.stream("windowdemo");
		source.flatMapValues(value-> Arrays.asList(value.toString().split("\\s+")))
				.map((x,y)->
				{
					return new KeyValue<String,String>(y,"1");
				}).groupByKey()
		.windowedBy(SessionWindows.with(Duration.ofSeconds(15).toMillis()))
				.count().toStream().foreach((x,y)->{
			System.out.println("x:"+x+" y:"+y);
		});



		final Topology topo = builder.build();
		final KafkaStreams streams = new KafkaStreams(topo,prop);

		final CountDownLatch latch = new CountDownLatch(1);
		Runtime.getRuntime().addShutdownHook(new Thread("stream"){
			@Override
			public void run() {
				streams.close();
				latch.countDown();
			}
		});
		streams.start();
		try {
			latch.await();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}

		System.exit(0);
	}
}

  • 运行Java进程
  • 在windowdemo生产消息
hello java
  • 对应的在windowdemo消费到了消息

在这里插入图片描述
注意:每次编写Java类时,application_id_config的名字都要不一样。

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值