【Kafka】Java整合Kafka

1.引入依赖

        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>2.3.1</version>
        </dependency>

2.搭建生产者

package com.wen.kafka;

import org.apache.kafka.clients.producer.*;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;
import java.util.concurrent.ExecutionException;

public class MyProducer {

    public static void main(String[] args) throws ExecutionException, InterruptedException {
        //配置信息
        Properties prop = new Properties();
        prop.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.117.80:9092");
        prop.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        prop.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

        //创建生产者
        Producer<String,String> producer = new KafkaProducer<String, String>(prop);

        //创建消息
        ProducerRecord<String,String> record = new ProducerRecord<>("test", "hello kafka-client");

        //同步发送消息
//        RecordMetadata metadata = producer.send(record).get();
//        System.out.println("同步消息——topic:"+metadata.topic()+"partition"+metadata.partition()+"offset"+metadata.offset());

        //异步发送消息
        producer.send(record, new Callback() {
            @Override
            public void onCompletion(RecordMetadata recordMetadata, Exception e) {
                if (e != null) {
                    System.out.println(e.getMessage());
                }
                if (recordMetadata != null) {
                    System.out.println("异步消息——topic:"+recordMetadata.topic()+"partition"+recordMetadata.partition()+"offset"+recordMetadata.offset());
                }
            }
        });
        Thread.sleep(1000);
    }
}

3.搭建消费者

package com.wen.kafka;

import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;

public class MyConsumer {

    public static void main(String[] args) {
        //参数信息
        Properties prop = new Properties();
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.117.80:9092");
        prop.put(ConsumerConfig.GROUP_ID_CONFIG,"testGroup");
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class.getName());

        //创建消费者
        Consumer<String,String> consumer = new KafkaConsumer<String, String>(prop);

        //订阅主题
        consumer.subscribe(Arrays.asList("test"));

        //拉取消息
        while (true){
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
            for (ConsumerRecord<String, String> record : records) {
                System.out.println(record.value());
            }
        }
    }
}

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
使用Spark Streaming整合Kafka可以实现实时流式数据处理。下面是一个简单的Java代码示例: ```java import org.apache.spark.SparkConf; import org.apache.spark.streaming.Duration; import org.apache.spark.streaming.api.java.JavaInputDStream; import org.apache.spark.streaming.api.java.JavaStreamingContext; import org.apache.spark.streaming.kafka010.ConsumerStrategies; import org.apache.spark.streaming.kafka010.KafkaUtils; import org.apache.spark.streaming.kafka010.LocationStrategies; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; public class KafkaSparkStreamingExample { public static void main(String[] args) throws InterruptedException { String brokers = "localhost:9092"; String groupId = "group1"; String topics = "topic1"; // Create context with a 2 seconds batch interval SparkConf sparkConf = new SparkConf().setAppName("KafkaSparkStreamingExample"); JavaStreamingContext streamingContext = new JavaStreamingContext(sparkConf, new Duration(2000)); // Create Kafka parameters map Map<String, Object> kafkaParams = new HashMap<>(); kafkaParams.put("bootstrap.servers", brokers); kafkaParams.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); kafkaParams.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); kafkaParams.put("group.id", groupId); kafkaParams.put("auto.offset.reset", "latest"); kafkaParams.put("enable.auto.commit", false); Collection<String> topicsSet = Arrays.asList(topics.split(",")); // Create direct kafka stream JavaInputDStream<String> messages = KafkaUtils.createDirectStream( streamingContext, LocationStrategies.PreferConsistent(), ConsumerStrategies.<String, String>Subscribe(topicsSet, kafkaParams) ).map(record -> record.value()); // Process each message in stream messages.foreachRDD(rdd -> { rdd.foreach(message -> System.out.println(message)); }); // Start the computation streamingContext.start(); streamingContext.awaitTermination(); } } ``` 在这个示例中,我们首先定义了Kafka的broker地址、消费者组ID和要消费的主题。然后,我们创建了一个Spark Streaming的JavaStreamingContext对象,并定义了每个批次的时间间隔为2秒。 接下来,我们定义了Kafka参数Map和要消费的主题集合,并使用KafkaUtils.createDirectStream()方法创建了一个JavaInputDStream对象。 最后,我们对每个批次中的每个消息进行处理,将其打印到控制台上。最后,我们启动了Spark Streaming上下文并等待它终止。 这只是一个简单的示例,你可以根据自己的需求进行修改和扩展。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值