Java实现kafka消息发送（仅做入门）

1. 小声哔哔

    kafka消息发送有三种方式，发送并忘记、同步发送和异步发送，下面直接使用代码演示，当然这里的实现代码仅做入门学习参考，真正的生产环境必然涉及多线程生产消息，配置统一规划等情况。

2. 前期准备

    首先写一个简单的消费者代码作为结果验证：

package com.coline.middleware.kafka.consumer;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.time.Duration;
import java.util.Collections;
import java.util.Properties;

/**
 * @author: Coline
 * @ClassName: KafkaConsumer 
 * @Date: 2020/3/7 22:16
 * @Description: 消费者代码，用于验证
 */
public class KafkaConsumer {
    public static void main(String[] args) {
        Properties properties = new Properties();
        // kafka群组集群地址
        properties.put("bootstrap.servers", "192.168.202.131:9092,192.168.202.131:9094,192.168.202.131:9095");
        properties.put("key.deserializer", StringDeserializer.class);
        properties.put("value.deserializer", StringDeserializer.class);

        // 消费群组
        properties.put(ConsumerConfig.GROUP_ID_CONFIG, "test1");
        org.apache.kafka.clients.consumer.KafkaConsumer<String, String> consumer = new org.apache.kafka.clients.consumer.KafkaConsumer<>(properties);
        try {
            consumer.subscribe(Collections.singletonList("TestTopic"));
            while (true) {
                ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(500));
                for (ConsumerRecord<String, String> record : records) {
                    System.out.println(String.format("topic:%s,分区：%d,偏移量：%d," + "key:%s,value:%s", record.topic(), record.partition(),
                            record.offset(), record.key(), record.value()));
                }
            }
        } finally {
            consumer.close();
        }
    }
}

   3. 发送并忘记

    代码如下：

package com.coline.middleware.kafka.nospring.sendtype;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.util.Properties;

/**
 * @author: Coline
 * @ClassName: SendMessageThenForget
 * @Date: 2020/3/7 22:16
 * @Description: 以发送并忘记的方式发送消息
 */
public class SendMessageThenForget {
    public static void main(String[] args) {
        Properties properties = new Properties();
        // kafka群组集群地址
        properties.put("bootstrap.servers", "192.168.202.131:9092,192.168.202.131:9094,192.168.202.131:9095");
        properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        KafkaProducer<String, String> producer = new KafkaProducer<>(properties);
        try {
            ProducerRecord<String, String> record;
            // 发送4条消息
            for (int i = 0; i < 4; i++) {
                record = new ProducerRecord<>("TestTopic", null, "Message: " + i);
                // 发送并发忘记
                producer.send(record);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            producer.close();
        }
    }
}

       消费者结果展示：

    代码中的生产消息只是简单的调用，将消息推送给kafka后就结束，但是若是有消费失败的情况还是会有默认的自动重试。

4. 异步发送

   代码如下：

package com.coline.middleware.kafka.nospring.sendtype;

import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import java.util.Properties;

/**
 * @author: Coline
 * @ClassName: KafkaAsynProducer
 * @Date: 2020/3/8 22:37
 * @Description: 异步模式发送kafka消息
 */
public class KafkaAsynProducer {
    private static KafkaProducer<String, String> producer = null;

    public static void main(String[] args) {
        Properties properties = new Properties();
        properties.put("bootstrap.servers", "192.168.202.131:9092,192.168.202.131:9094,192.168.202.131:9095");
        properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        producer = new KafkaProducer<>(properties);

        try {
            for (int i = 0; i < 4; i++) {
                producer.send(new ProducerRecord<>("TestTopic", "test", "Message: " + i), new Callback() {
                    @Override
                    public void onCompletion(RecordMetadata metadata, Exception exception) {
                        if (null != exception) {
                            exception.printStackTrace();
                        }
                        if (null != metadata) {
                            System.out.println("offset: " + metadata.offset() + " "
                                    + "partition: " + metadata.partition());
                        }
                    }
                });
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            producer.close();
        }

    }
}

   消息生产后返回信息：

    消费者结果展示：

    可以看到，我们调用的KafkaProducer的send(ProducerRecord<K, V> record, Callback callback)方法，该方法可以根据Callback中实现的onCompletion方法来获取消息生产是发生的异常和消息发送时的返回
    多说一句，new ProducerRecord的参数分别为topic，key和message，可以看到代码中我设置了key值，这样可以保证相同key的消息都会放入一个partition，根据kafka的机制，同一个broker的同一个partition下的消息有序，这样就能够保证消费的时候消息是有序的。

5. 同步发送

    代码如下：

package com.coline.middleware.kafka.nospring.sendtype;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import java.util.Properties;
import java.util.concurrent.Future;

/**
 * @author: Coline
 * @ClassName: KafkaSyncProducer
 * @Date: 2020/3/8 23:48
 * @Description: 同步模式发送kafka消息
 */
public class KafkaSyncProducer {
    private static KafkaProducer<String, String> producer = null;

    public static void main(String[] args) {
        Properties properties = new Properties();
        properties.put("bootstrap.servers", "192.168.202.131:9092,192.168.202.131:9094,192.168.202.131:9095");
        properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        producer = new KafkaProducer<>(properties);

        try {
            for (int i = 0; i < 4; i++) {
                Future<RecordMetadata> future = producer.send(new ProducerRecord<>("TestTopic", "test", "Message: " + i));
                //阻塞在这个位置
                RecordMetadata recordMetadata = future.get();
                if (null != recordMetadata) {
                    System.out.println("offset:" + recordMetadata.offset() + "-" + "partition:" + recordMetadata.partition());
                }
            }

        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            producer.close();
        }
    }
}

    消息生产后返回：

    消息消费结果展示：

    看到代码中是利用Future实现同步等待结果返回，这个就是kafka生产的同步模式

6. 总结

    发送并忘记的模式对资源占用少，效率高，但是没有结果返回，不利于信息记录，同步发送存在阻塞的情况，所以生产环境中使用异步发送模式较多。