kafka Java例子

版本 2.11-1.0.0

1.启动远程服务

在linux中启动kafka 参见kafka Quickstart
默认情况下，kafka不允许跨ip访问，因此本地通过Eclipse跑Java代码访问远程服务时，需要开放端口
修改config/server.properties，加入一条

advertised.listeners=PLAINTEXT://10.43.39.215:9092

如果没有配置，会报如下错误信息

org.apache.kafka.common.errors.TimeoutException: Expiring 7 record(s) for topic1-0: 30032 ms has passed since batch creation plus linger time

本文的例子来自于源码 kafka-0.10.0.0\examples\src\main\java\kafka\examples，为了方便理解，稍微简化改造了一下

2.修改日志

kafka的client包默认采用log4j，如果没有配置，会有提示

log4j:WARN No appenders could be found for logger (org.apache.kafka.clients.consumer.ConsumerConfig).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.

虽然不影响java代码运行，但是没有配置后，默认的日志级别是DEBUG，会严重影响观察程序的运行情况。
在工程的path下放入log4j.properties

log4j.rootLogger=INFO,Console,A

log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d [%t] %-5p [%c] - %m%n

3.Producer

package kafka.v2_1_1and1_0_0;

import java.util.Properties;
import java.util.concurrent.ExecutionException;

import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class Producer extends Thread {
    private static Logger logger = LoggerFactory.getLogger(Producer.class);

    private final KafkaProducer<Integer, String> producer;
    private final String topic;
    private final Boolean isAsync;

    public Producer(String topic, Boolean isAsync) {
        Properties props = new Properties();
        // 集群地址，多个服务器用"，"分隔
        props.put("bootstrap.servers", "10.43.39.215:9092");
        props.put("client.id", "DemoProducer");
        // key、value的序列化，此处以字符串为例，使用kafka已有的序列化类
        props.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        producer = new KafkaProducer(props);
        this.topic = topic;
        this.isAsync = isAsync;
    }

    public void run() {
        int messageNo = 1;
        while (true) {
            String messageStr = "Message_" + messageNo;
            long startTime = System.currentTimeMillis();
            if (isAsync) { // Send asynchronously
                producer.send(new ProducerRecord(topic, messageNo, messageStr), new DemoCallBack(
                        startTime, messageNo, messageStr));
            } else { // Send synchronously
                try {
                    producer.send(new ProducerRecord(topic, messageNo, messageStr)).get();
                    System.out.println("Sent message: (" + messageNo + ", " + messageStr + ")");
                } catch (InterruptedException | ExecutionException e) {
                    e.printStackTrace();
                }
            }
            ++messageNo;

            try {
                Thread.sleep(5000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        Producer producerThread = new Producer(KafkaProperties.TOPIC, true);
        producerThread.start();

    }

}

class DemoCallBack implements Callback {

    private final long startTime;
    private final int key;
    private final String message;

    public DemoCallBack(long startTime, int key, String message) {
        this.startTime = startTime;
        this.key = key;
        this.message = message;
    }

    /**
     * A callback method the user can implement to provide asynchronous handling of request
     * completion. This method will be called when the record sent to the server has been
     * acknowledged. Exactly one of the arguments will be non-null.
     *
     * @param metadata
     *            The metadata for the record that was sent (i.e. the partition and offset). Null if
     *            an error occurred.
     * @param exception
     *            The exception thrown during processing of this record. Null if no error occurred.
     */
    public void onCompletion(RecordMetadata metadata, Exception exception) {
        long elapsedTime = System.currentTimeMillis() - startTime;
        if (metadata != null) {
            System.out.println("message(" + key + ", " + message + ") sent to partition("
                    + metadata.partition() + "), " + "offset(" + metadata.offset() + ") in "
                    + elapsedTime + " ms");
        } else {
            exception.printStackTrace();
        }
    }
}

执行main方法：

2019-11-22 18:57:49,633 [main] INFO [org.apache.kafka.common.utils.AppInfoParser] - Kafka version : 1.0.0
2019-11-22 18:57:49,633 [main] INFO [org.apache.kafka.common.utils.AppInfoParser] - Kafka commitId : aaa7af6d4a11b29d
message(1, Message_1) sent to partition(0), offset(113) in 261 ms
message(2, Message_2) sent to partition(0), offset(114) in 4 ms

3.Consumer

package kafka.v2_1_1and1_0_0;

import java.util.Collections;
import java.util.Properties;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

public class Consumer extends Thread {
    private final KafkaConsumer<Integer, String> consumer;
    private final String topic;

    public Consumer(String topic) {
        Properties props = new Properties();
        // 集群地址，多个地址用"，"分隔
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.43.39.215:9092");
        // 设置消费者的group id
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "DemoConsumer");
        // 如果为真，consumer所消费消息的offset将会自动的同步到zookeeper。如果消费者死掉时，由新的consumer使用继续接替
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
        // consumer向zookeeper提交offset的频率
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
                "org.apache.kafka.common.serialization.IntegerDeserializer");
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
                "org.apache.kafka.common.serialization.StringDeserializer");

        consumer = new KafkaConsumer(props);
        this.topic = topic;
    }

    @Override
    public void run() {
        consumer.subscribe(Collections.singletonList(this.topic));

        while (true) {
            ConsumerRecords<Integer, String> records = consumer.poll(1000);
            for (ConsumerRecord<Integer, String> record : records) {
                System.out.println("Received message: (" + record.key() + ", " + record.value()
                        + ") at offset " + record.offset());
            }

        }
    }

    public static void main(String[] args) {

        Consumer consumerThread = new Consumer(KafkaProperties.TOPIC);
        consumerThread.start();
    }

执行main方法：

auto.commit.interval.ms = 1000
auto.offset.reset = latest
bootstrap.servers = [10.43.39.215:9092]
check.crcs = true
client.id =
connections.max.idle.ms = 540000
enable.auto.commit = true
exclude.internal.topics = true
fetch.max.bytes = 52428800
fetch.max.wait.ms = 500
fetch.min.bytes = 1
group.id = DemoConsumer
heartbeat.interval.ms = 3000
interceptor.classes = null
internal.leave.group.on.close = true
isolation.level = read_uncommitted
key.deserializer = class org.apache.kafka.common.serialization.IntegerDeserializer
max.partition.fetch.bytes = 1048576
max.poll.interval.ms = 300000
max.poll.records = 500
metadata.max.age.ms = 300000
metric.reporters = []
metrics.num.samples = 2
metrics.recording.level = INFO
metrics.sample.window.ms = 30000
partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor]
receive.buffer.bytes = 65536
reconnect.backoff.max.ms = 1000
reconnect.backoff.ms = 50
request.timeout.ms = 305000
retry.backoff.ms = 100
sasl.jaas.config = null
sasl.kerberos.kinit.cmd = /usr/bin/kinit
sasl.kerberos.min.time.before.relogin = 60000
sasl.kerberos.service.name = null
sasl.kerberos.ticket.renew.jitter = 0.05
sasl.kerberos.ticket.renew.window.factor = 0.8
sasl.mechanism = GSSAPI
security.protocol = PLAINTEXT
send.buffer.bytes = 131072
session.timeout.ms = 30000
ssl.cipher.suites = null
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
ssl.endpoint.identification.algorithm = null
ssl.key.password = null
ssl.keymanager.algorithm = SunX509
ssl.keystore.location = null
ssl.keystore.password = null
ssl.keystore.type = JKS
ssl.protocol = TLS
ssl.provider = null
ssl.secure.random.implementation = null
ssl.trustmanager.algorithm = PKIX
ssl.truststore.location = null
ssl.truststore.password = null
ssl.truststore.type = JKS
value.deserializer = class org.apache.kafka.common.serialization.StringDeserializer
2019-11-22 19:00:52,533 [Thread-0] INFO [org.apache.kafka.clients.consumer.internals.AbstractCoordinator] - [Consumer clientId=consumer-1, groupId=DemoConsumer] Discovered coordinator 10.43.39.215:9092 (id: 2147483647 rack: null)
2019-11-22 19:00:52,538 [Thread-0] INFO [org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] - [Consumer clientId=consumer-1, groupId=DemoConsumer] Revoking previously assigned partitions []
2019-11-22 19:00:52,538 [Thread-0] INFO [org.apache.kafka.clients.consumer.internals.AbstractCoordinator] - [Consumer clientId=consumer-1, groupId=DemoConsumer] (Re-)joining group
2019-11-22 19:00:52,564 [Thread-0] INFO [org.apache.kafka.clients.consumer.internals.AbstractCoordinator] - [Consumer clientId=consumer-1, groupId=DemoConsumer] Successfully joined group with generation 9
2019-11-22 19:00:52,564 [Thread-0] INFO [org.apache.kafka.clients.consumer.internals.ConsumerCoordinator] - [Consumer clientId=consumer-1, groupId=DemoConsumer] Setting newly assigned partitions [topic1-0]
Received message: (1, Message_1) at offset 107
Received message: (2, Message_2) at offset 108

日志的前半部分展示的是所有consumer的参数，方便我们定位问题和了解默认值，比如我们手动设置了props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, “1000”);映射auto.commit.interval.ms = 1000，那么不在Consumer.java中出现的参数就是默认值，同理，Producer.java也一样，本文没有展示