Kafka-多线程处理记录

最新推荐文章于 2023-04-21 00:30:00 发布

山沉

最新推荐文章于 2023-04-21 00:30:00 发布

阅读量402

点赞数 2

分类专栏：中间件文章标签： kafka big data java

本文链接：https://blog.csdn.net/qq_43925043/article/details/120754465

版权

中间件专栏收录该内容

1 篇文章 0 订阅

订阅专栏

本文详细分析了一个使用Java实现的多线程Kafka消费者示例，该消费者手动提交偏移量并处理消费组再平衡。关键点包括确保记录只被处理一次，提交处理后的偏移量，以及在分区重分配时避免重复处理。同时，文章讨论了如何处理处理速度慢、消息异常和线程间的协调问题，提出了线程暂停和恢复的策略来保证处理顺序。此外，还提醒注意避免因处理时间过长导致的消费者组移除问题。

摘要由CSDN通过智能技术生成

Kafka-多线程代码赏析

在2020年8月 13号 IGOR BUZATOVIĆ 这个人在

https://www.confluent.io/blog/kafka-consumer-multi-threaded-messaging/写下了这篇博客。

以下内容，纯属学习。

源代码路径

https://github.com/inovatrend/mtc-demo

MultithreadedKafkaConsumer


package com.inovatrend.mtcdemo;



import org.apache.kafka.clients.consumer.*;

import org.apache.kafka.common.TopicPartition;

import org.apache.kafka.common.errors.WakeupException;

import org.apache.kafka.common.serialization.StringDeserializer;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.time.Duration;

import java.time.temporal.ChronoUnit;

import java.util.*;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.atomic.AtomicBoolean;



public class MultithreadedKafkaConsumer implements Runnable, ConsumerRebalanceListener {



    private final KafkaConsumer<String, String> consumer;

    private final ExecutorService executor = Executors.newFixedThreadPool(8);

    private final Map<TopicPartition, Task> activeTasks = new HashMap<>();

    private final Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>();

    private final AtomicBoolean stopped = new AtomicBoolean(false);

    private long lastCommitTime = System.currentTimeMillis();

    private final Logger log = LoggerFactory.getLogger(MultithreadedKafkaConsumer.class);





    public MultithreadedKafkaConsumer(String topic) {

        Properties config = new Properties();

        config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");

        config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

        config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

        config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);

        config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

        config.put(ConsumerConfig.GROUP_ID_CONFIG, "multithreaded-consumer-demo");

        consumer = new KafkaConsumer<>(config);

        new Thread(this).start();

    }





    @Override

    public void run() {

        try {

            consumer.subscribe(Collections.singleton("topic-name"), this);

            while (!stopped.get()) {

                ConsumerRecords<String, String> records = consumer.poll(Duration.of(100, ChronoUnit.MILLIS));

                handleFetchedRecords(records);

                checkActiveTasks();

                commitOffsets();

            }

        } catch (WakeupException we) {

            if (!stopped.get())

                throw we;

        } finally {

            consumer.close();

        }

    }





    private void handleFetchedRecords(ConsumerRecords<String, String> records) {

        if (records.count() > 0) {

            List<TopicPartition> partitionsToPause = new ArrayList<>();

             records.partitions().forEach(partition -> {

                 List<ConsumerRecord<String, String>> partitionRecords = records.records(partition);

                 Task task = new Task(partitionRecords);

                 partitionsToPause.add(partition);

                 executor.submit(task);

                 activeTasks.put(partition, task);

             });

            consumer.pause(partitionsToPause);

        }

    }



    private void commitOffsets() {

        try {

            long currentTimeMillis = System.currentTimeMillis();

            if (currentTimeMillis - lastCommitTime > 5000) {

                if(!offsetsToCommit.isEmpty()) {

                    consumer.commitSync(offsetsToCommit);

                    offsetsToCommit.clear();

                }

                lastCommitTime = currentTimeMillis;

            }

        } catch (Exception e) {

            log.error("Failed to commit offsets!", e);

        }

    }





    private void checkActiveTasks() {

        List<TopicPartition> finishedTasksPartitions = new ArrayList<>();

        activeTasks.forEach((partition, task) -> {

            if (task.isFinished())

                finishedTasksPartitions.add(partition);

            long offset = task.getCurrentOffset();

            if (offset > 0)

                offsetsToCommit.put(partition, new OffsetAndMetadata(offset));

        });

        finishedTasksPartitions.forEach(partition -> activeTasks.remove(partition));

        consumer.resume(finishedTasksPartitions);

    }





    @Override

    public void onPartitionsRevoked(Collection<TopicPartition> partitions) {



        // 1. Stop all tasks handling records from revoked partitions

        Map<TopicPartition, Task> stoppedTask = new HashMap<>();

        for (TopicPartition partition : partitions) {

            Task task = activeTasks.remove(partition);

            if (task != null) {

                task.stop();

                stoppedTask.put(partition, task);

            }

        }



        // 2. Wait for stopped tasks to complete processing of current record

        stoppedTask.forEach((partition, task) -> {

            long offset = task.waitForCompletion();

            if (offset > 0)

                offsetsToCommit.put(partition, new OffsetAndMetadata(offset));

        });





        // 3. collect offsets for revoked partitions

        Map<TopicPartition, OffsetAndMetadata> revokedPartitionOffsets = new HashMap<>();

        partitions.forEach( partition -> {

            OffsetAndMetadata offset = offsetsToCommit.remove(partition);

            if (offset != null)

                revokedPartitionOffsets.put(partition, offset);

        });



        // 4. commit offsets for revoked partitions

        try {

            consumer.commitSync(revokedPartitionOffsets);

        } catch (Exception e) {

            log.warn("Failed to commit offsets for revoked partitions!");

        }

    }



    @Override

    public void onPartitionsAssigned(Collection<TopicPartition> partitions) {

        consumer.resume(partitions);

    }





    public void stopConsuming() {

        stopped.set(true);

        consumer.wakeup();

    }



}

Task

下面赏析线程代码




package com.inovatrend.mtcdemo;



import org.apache.kafka.clients.consumer.ConsumerRecord;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.util.List;

import java.util.concurrent.CompletableFuture;

import java.util.concurrent.ExecutionException;

import java.util.concurrent.atomic.AtomicLong;

import java.util.concurrent.locks.ReentrantLock;





public class Task implements Runnable {



    private final List<ConsumerRecord<String, String>> records;



    private volatile boolean stopped = false;



    private volatile boolean started = false;



    private volatile boolean finished = false;



    private final CompletableFuture<Long> completion = new CompletableFuture<>();



    private final ReentrantLock startStopLock = new ReentrantLock();



    private final AtomicLong currentOffset = new AtomicLong();



    private Logger log = LoggerFactory.getLogger(Task.class);



    public Task(List<ConsumerRecord<String, String>> records) {

        this.records = records;

    }





    public void run() {

        startStopLock.lock();

        if (stopped){

            return;

        }

        started = true;

        startStopLock.unlock();



        for (ConsumerRecord<String, String> record : records) {

            if (stopped)

                break;

            // process record here and make sure you catch all exceptions;

            currentOffset.set(record.offset() + 1);

        }

        finished = true;

        completion.complete(currentOffset.get());

    }



    public long getCurrentOffset() {

        return currentOffset.get();

    }



    public void stop() {

        startStopLock.lock();

        this.stopped = true;

        if (!started) {

            finished = true;

            completion.complete(currentOffset.get());

        }

        startStopLock.unlock();

    }



    public long waitForCompletion() {

        try {

            return completion.get();

        } catch (InterruptedException | ExecutionException e) {

            return -1;

        }

    }



    public boolean isFinished() {

        return finished;

    }



}