KafkaConsumer高阶Api消费Util

使用Kafka高阶Api，基于通用业务场景封装基于Kafka0.8版本的消费工具类。

项目背景：

运行任务的监控数据通过kafka上报，应用中需要消费Topic中的数据。

功能介绍：

工具类有如下特性

1. ConsumerGroup提供通用的Topic消费服务
2. 以Builder模式配置ZK、Topic、GroupId、消费线程数、线程池大小
3. 支持复用现有线程池，将消费任务（业务逻辑）放入现有线程池
4. 支持消费任务的shutdown（同步关闭内部线程池）

实现类(ConsumerGroup.java)：

package com.wj.kafka.consumer.high;

import com.wj.kafka.consumer.TaskInterface;
import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;

import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * 消费组工具类，可设置消费线程数和停止消费
 *
 */
public class ConsumerGroup {

    /**
     * 待消费的Topic
     */
    private final String topic;
    /**
     * 消费线程数
     */
    private final int numOfConsumer;
    /**
     * ConsumerConnector
     */
    private ConsumerConnector consumerConnector;
    /**
     * 线程池服务，提供多线程消费
     */
    private ExecutorService executor;

    /**
     * 构建消费组工具类
     *
     * @param consumerConnector ConsumerConnector
     * @param topic             待消费Topic
     * @param numOfConsumer     消费池大小
     */
    private ConsumerGroup(ConsumerConnector consumerConnector, String topic, int numOfConsumer) {
        this.consumerConnector = consumerConnector;
        this.topic = topic;
        this.numOfConsumer = numOfConsumer;
    }

    /**
     * 构建KafkaConsumerGroupBuilder
     *
     * @return KafkaConsumerGroupBuilder
     */
    public static KafkaConsumerGroupBuilder instance() {
        return new ConsumerGroup.KafkaConsumerGroupBuilder().instance();
    }

    /**
     * 提交任务
     *
     * @param task 业务逻辑
     */
    public void submit(TaskInterface task, int numOfThread) {
        // partition流
        List<KafkaStream<byte[], byte[]>> streams = getKafkaStreams();

        // 新建线程池
        this.executor = Executors.newFixedThreadPool(numOfThread);

        submitTasks(this.executor, streams, task);
    }

    /**
     * 提交线程消费任务，使用提供的线程池executor
     *
     * @param task     业务逻辑
     * @param executor 线程服务
     */
    public void submit(TaskInterface task, ExecutorService executor) {
        // partition流
        List<KafkaStream<byte[], byte[]>> streams = getKafkaStreams();

        submitTasks(executor, streams, task);
    }

    /**
     * 提交消费任务
     * @param executor 线程池服务
     * @param streams 各分区数据流
     * @param task 业务逻辑
     */
    private void submitTasks(ExecutorService executor, List<KafkaStream<byte[], byte[]>> streams, TaskInterface task) {
        for (KafkaStream<byte[], byte[]> stream : streams) {
            executor.submit(new BusinessTask(stream, task));
        }
    }

    /**
     * 关闭任务
     */
    public void shutdown() {
        if (consumerConnector != null) {
            consumerConnector.shutdown();
        }
        if (executor != null) {
            executor.shutdown();
            try {
                if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
                    System.out.println("Timed out waiting for consumer threads to shut down, exiting uncleanly");
                }
            } catch (InterruptedException e) {
                System.out.println("Interrupted during shutdown, exiting uncleanly");
            }
        }
    }

    /**
     * 获取topic的输入流 KafkaStream
     *
     * @return topic分区数据流
     */
    private List<KafkaStream<byte[], byte[]>> getKafkaStreams() {
        Map<String, Integer> topicCountMap = new HashMap<>(2);
        topicCountMap.put(topic, numOfConsumer);
        return this.consumerConnector.createMessageStreams(topicCountMap).get(topic);
    }

    /**
     * ConsumerGroup构造器
     */
    public static class KafkaConsumerGroupBuilder {
        /**
         * ZK地址
         */
        private String zookeeper;
        /**
         * topic主题
         */
        private String topic;
        /**
         * GroupId
         */
        private String groupId;
        /**
         * 消费线程数
         */
        private Integer numOfConsumer;
        /**
         * offset提交间隔
         */
        private int commitInterval;

        private KafkaConsumerGroupBuilder() {
        }

        KafkaConsumerGroupBuilder instance() {
            return new KafkaConsumerGroupBuilder();
        }

        public KafkaConsumerGroupBuilder setBaseInfo(String zookeeper, String topic, String groupId) {
            this.zookeeper = zookeeper;
            this.topic = topic;
            this.groupId = groupId;
            return this;
        }

        public KafkaConsumerGroupBuilder setNumOfConsumer(int numOfConsumer) {
            this.numOfConsumer = numOfConsumer;
            return this;
        }

        public KafkaConsumerGroupBuilder setCommitInterval(int commitInterval) {
            this.commitInterval = commitInterval;
            return this;
        }

        /**
         * 构造ConsumerGroup对象
         *
         * @return ConsumerGroup
         */
        public ConsumerGroup builder() {
            Objects.requireNonNull(zookeeper, "zookeeper address should be set.");
            Objects.requireNonNull(groupId, "groupId should be set.");
            ConsumerConnector consumer = Consumer.createJavaConsumerConnector(createConsumerConfig(zookeeper, groupId));

            Objects.requireNonNull(topic, "topic should be set.");
            Objects.requireNonNull(numOfConsumer, "numOfConsumer should be set.");
            return new ConsumerGroup(consumer, topic, numOfConsumer);
        }

        /**
         * 创建ConsumerConfig
         *
         * @param zookeeper ZK地址
         * @param groupId   GroupId
         * @return ConsumerConfig
         */
        private ConsumerConfig createConsumerConfig(String zookeeper, String groupId) {
            Properties props = new Properties();
            props.put("zookeeper.connect", zookeeper);
            props.put("group.id", groupId);
            props.put("zookeeper.session.timeout.ms", "400");
            // zk follower落后leader的时间
            props.put("zookeeper.sync.time.ms", "200");
            // 自动提交时间
            props.put("auto.commit.interval.ms", String.valueOf(commitInterval));
            return new ConsumerConfig(props);
        }
    }

    public static class BusinessTask implements Runnable {

        private KafkaStream<byte[], byte[]> kafkaStream;

        private TaskInterface task;

        private BusinessTask(KafkaStream<byte[], byte[]> kafkaStream, TaskInterface task) {
            this.kafkaStream = kafkaStream;
            this.task = task;
        }

        @Override
        public void run() {
            ConsumerIterator<byte[], byte[]> it = this.kafkaStream.iterator();
            while (it.hasNext()) {
                task.doTask(new String(it.next().message()));
            }

            System.out.println("Shutting down Thread : " + Thread.currentThread().getName());
        }
    }
}

Task接口类

@FunctionalInterface
public interface TaskInterface {

    /**
     * 任务操作
     * @param msg 一条消息
     */
    void doTask(String msg);

}

测试类(TestMain.java)：

package com.wj.kafka.consumer.high;

import com.wj.kafka.consumer.Constants;
import com.wj.kafka.consumer.TaskInterface;

/**
 * 测试Kafka高阶API工具类
 *
 * @author wj
 */
public class TestMain implements Constants {

    private static TaskInterface task = e -> System.out.println("msg : " + e);

    public static void main(String[] args) throws InterruptedException {
        ConsumerGroup consumerGroup = ConsumerGroup
                .instance()
                .setBaseInfo(ZK_LIST, TOPIC, GROUP_ID)
                .setCommitInterval(1000)
                .setNumOfConsumer(2).builder();

        consumerGroup.submit(task, 2);

        Thread.sleep(5000);

//        consumerGroup.shutdown();
    }
}

参考：https://www.cnblogs.com/cyfonly/p/5954614.html