Kafka 简单部署及Java测试

快速部署

资料来源：https://kafka.apache.org/quickstart

1. 下载

https://kafka.apache.org/downloads

---

实践：

> https://www.apache.org/dyn/closer.cgi?path=/kafka/2.2.0/kafka_2.12-2.2.0.tgz
> tar -xzf kafka_2.12-2.2.0.tgz
> cd kafka_2.12-2.2.0

2. 启动服务器

Kafka使用ZooKeeper，因此如果您还没有ZooKeeper服务器，则需要先启动它。
您可以使用与kafka一起打包的便捷脚本来获得快速测试的单节点ZooKeeper实例。启动如下：

bin/zookeeper-server-start.sh config/zookeeper.properties

启动Kafka服务器：

bin/kafka-server-start.sh config/server.properties

3. 主题

让我们创建一个名为“test”的主题，它只包含一个分区，只有一个副本：

bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test
或
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic [topicName]

如果我们运行list topic命令，我们现在可以看到该主题：

bin/kafka-topics.sh --list --bootstrap-server localhost:9092
test

bin/kafka-topics.sh --list --zookeeper localhost:2181

4. 消息提供者

Kafka附带一个命令行客户端（kafka-console-producer.sh），它将从文件或标准输入中获取输入，并将其作为消息发送到Kafka集群。默认情况下，每行将作为单独的消息发送。

运行生产者，然后在控制台中键入一些消息以发送到服务器。

bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
This is a message
This is another message

5. 消息消费者

Kafka还有一个命令行使用者，它会将消息转储到标准输出。

bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
This is a message
This is another message

6. Kafka集群

首先，我们为每个代理创建一个配置文件：

cp config/server.properties config/server-1.properties
cp config/server.properties config/server-2.properties

现在编辑这些新文件并设置以下属性：

config/server-1.properties:

    broker.id=1
    listeners=PLAINTEXT://:9093
    log.dirs=/tmp/kafka-logs-1

config/server-2.properties:

    broker.id=2
    listeners=PLAINTEXT://:9094
    log.dirs=/tmp/kafka-logs-2

该broker.id属性是群集中每个节点的唯一且永久的名称。

启动两个新节点：

bin/kafka-server-start.sh config/server-1.properties &
…
bin/kafka-server-start.sh config/server-2.properties &
…

现在创建一个复制因子为3的新主题：

bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 3 --partitions 1 --topic my-replicated-topic
或
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic [topicName]

显示主题描述信息

bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic [topicName]

Java 实践

代码来源：https://github.com/apache/kafka/tree/trunk/examples/src/main/java/kafka/examples

1. 配置

public class KafkaProperties {
    public static final String TOPIC = "test1";
    public static final String KAFKA_SERVER_URL = "localhost";
    public static final int KAFKA_SERVER_PORT = 9092;
    private KafkaProperties() {
    }
}

2. 消息生产者

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.IntegerSerializer;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;
import java.util.concurrent.ExecutionException;

public class Producer extends Thread {
    private final KafkaProducer<Integer, String> producer;
    private final String topic;
    private final Boolean isAsync;

    public Producer(String topic, Boolean isAsync) {
        Properties props = new Properties();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
        props.put(ProducerConfig.CLIENT_ID_CONFIG, "DemoProducer");
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        producer = new KafkaProducer<>(props);
        this.topic = topic;
        this.isAsync = isAsync;
    }

    @Override
    public void run() {
        int messageNo = (int) System.currentTimeMillis();
        String messageStr = "Message_" + messageNo;

        try {
            producer.send(new ProducerRecord(topic,
                    messageNo,
                    messageStr)).get();
            System.out.println("Sent message: (" + messageNo + ", " + messageStr + ")");
        } catch (InterruptedException | ExecutionException e) {
            e.printStackTrace();
        }
    }
}

public class ProducerDemo {

    public static void main(String[] args) throws InterruptedException {
        Producer producerThread = new Producer(KafkaProperties.TOPIC, false);
        producerThread.start();
        producerThread.join();
    }

}

3. 消息消费者

import kafka.utils.ShutdownableThread;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.time.Duration;
import java.util.Collections;
import java.util.Properties;

public class Consumer extends ShutdownableThread {
    private final KafkaConsumer<Integer, String> consumer;
    private final String topic;

    public Consumer(String topic) {
        super("KafkaConsumerExample", false);
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "DemoConsumer");
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.IntegerDeserializer");
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");

        consumer = new KafkaConsumer<>(props);
        this.topic = topic;
    }

    @Override
    public void doWork() {
        consumer.subscribe(Collections.singletonList(this.topic));
        ConsumerRecords<Integer, String> records = consumer.poll(Duration.ofSeconds(1));
        for (ConsumerRecord<Integer, String> record : records) {
            System.out.println("接收消息： message: (" + record.key() + ", " + record.value() + ") at offset " + record.offset());
        }
    }

    @Override
    public String name() {
        return null;
    }

    @Override
    public boolean isInterruptible() {
        return false;
    }
}

public class ConsumerDemo {

    public static void main(String[] args) {
        Consumer consumerThread = new Consumer(KafkaProperties.TOPIC);
        consumerThread.start();
    }
}

4. maven

<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka_2.12</artifactId>
    <version>2.2.1</version>
</dependency>