通过Java client访问Kafka

1. Install Kafka
1) download kafka binary from https://kafka.apache.org/downloads
2) extract binary

$ tar xvf kafka_2.13-3.3.1.tgz -C ~/bigdata/

2. Start Kafka
1) start zookeeper in daemon mode

$ cd ~/bigdata/kafka_2.13-3.3.1
$ bin/zookeeper-server-start.sh -daemon config/zookeeper.properties
$ netstat -lnpt | grep -i TCP | grep `jps | grep -w QuorumPeerMain | awk '{print $1}'` | grep "LISTEN"
tcp6       0      0 :::45835                :::*                    LISTEN      568684/java         
tcp6       0      0 :::2181                 :::*                    LISTEN      568684/java

2) start kafka server in daemon mode

$ bin/kafka-server-start.sh -daemon config/server.properties
$ netstat -lnpt | grep -i TCP | grep `jps | grep -w Kafka | awk '{print $1}'` | grep "LISTEN"
tcp6       0      0 :::33011                :::*                    LISTEN      569177/java         
tcp6       0      0 :::9092                 :::*                    LISTEN      569177/java

3. Test Kafka
1) create a topic

$ bin/kafka-topics.sh --create --topic test --bootstrap-server localhost:9092
Created topic test.
$ bin/kafka-topics.sh --describe --topic test --bootstrap-server localhost:9092
Topic: test	TopicId: oLdPl33IR7KZFGmrURMKFw	PartitionCount: 1	ReplicationFactor: 1	Configs: 
	Topic: test	Partition: 0	Leader: 0	Replicas: 0	Isr: 0

2) producer events

$ bin/kafka-console-producer.sh --topic test --bootstrap-server localhost:9092
>hello world
>good morning
>cheer
>...
input ^C to break

3) consumer events

$ bin/kafka-console-consumer.sh --topic test --from-beginning --bootstrap-server localhost:9092 
hello world
good morning
cheer
...
input ^C to break

4. Access Kafka from Java client
1) download kafka client binary from https://jar-download.com/artifacts/org.apache.kafka/kafka-clients/3.3.1

$ tar xvf kafka-client-3.3.1.tgz -C ~/learn/java/java8/lib/

2) Write the kafka Java client

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

public class Kafka {
    static class KfkProducer {
        Producer<String, String> producer;

        KfkProducer(String host, int port) {
            Properties props = new Properties();
            props.put("bootstrap.servers", String.format("%s:%d", host, port));
//            props.put("linger.ms", 1);
            props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
            props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
            producer = new KafkaProducer<>(props);
        }

        void close() {
            producer.close();
        }

        RecordMetadata send(String topic, String key, String value) {
            Future<RecordMetadata> result = producer.send(new ProducerRecord<>(topic, key, value));
            RecordMetadata meta = null;
            try {
                meta = result.get();
            } catch (InterruptedException | ExecutionException e) {
                e.printStackTrace();
            }
            return meta;
        }
    }

    static class KfkConsumer {
        KafkaConsumer<String, String> consumer;

        KfkConsumer(String host, int port, List<String> topics) {
            Properties props = new Properties();
            props.put("bootstrap.servers", String.format("%s:%d", host, port));
            props.put("group.id", "group01");
            props.put("enable.auto.commit", "true");
            props.put("auto.commit.interval.ms", "1000");
            props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            consumer = new KafkaConsumer<>(props);
            consumer.subscribe(topics);
        }

        void close() {
            consumer.close();
        }

        List<List<Object>> poll(int num) {
            List<List<Object>> result = new ArrayList<>();
            while (result.size() < num) {
                ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
                for (ConsumerRecord<String, String> record : records) {
                    result.add(Arrays.asList(record.offset(), record.key(), record.value()));
                }
            }
            return result;
        }
    }

    public static void main(String... argv) {
        KfkProducer producer = new KfkProducer("localhost", 9092);
        for (int i = 0; i < 5; i ++) {
            System.out.println(producer.send("test", "" + (i % 5), Integer.toString(i)));
        }
        producer.close();

        KfkConsumer consumer = new KfkConsumer("localhost", 9092, Arrays.asList("test"));
        List<List<Object>> records = consumer.poll(5);
        for (List<Object> record: records) {
            System.out.printf("offset = %d, key = %s, value = %s%n",
                    (long)(record.get(0)), record.get(1), record.get(2));
        }
        consumer.close();
    }
}

3) compile the client

$ javac -cp "lib/kafka-client-3.3.1/*" Kafka.java

4) run the client

$ java -cp "lib/kafka-client-3.3.1/*:." Kafka
test-0@8
test-0@9
test-0@10
test-0@11
test-0@12
offset = 8, key = 0, value = 0
offset = 9, key = 1, value = 1
offset = 10, key = 2, value = 2
offset = 11, key = 3, value = 3
offset = 12, key = 4, value = 4

Note: following scripts can be used to stop servers and clean all created events as need

$ bin/kafka-server-stop.sh
$ bin/zookeeper-server-stop.sh
$ rm -rf /tmp/kafka-logs /tmp/zookeeper /tmp/kraft-combined-logs

Reference: https://kafka.apache.org/quickstart