kafka多线程消费

最新推荐文章于 2024-07-22 16:17:17 发布

一一可可

最新推荐文章于 2024-07-22 16:17:17 发布

阅读量2.1k

点赞数 1

分类专栏： kafka 文章标签： kafka

本文链接：https://blog.csdn.net/weixin_41452575/article/details/117518279

版权

该博客主要介绍了如何配置和使用Kafka进行多线程消费，包括两种方案：方案一是直接的多线程消费，方案二是结合线程池进行消费。作者推荐采用方案二，并在启动类文件中添加了相关代码以实现多线程加线程池的消费方式。

摘要由CSDN通过智能技术生成

备注：采用方案二

kafka生产消费配置类

package com.hnu.common.kafka.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;

/**
 * @author yuanhx
 * @Date 2021/6/3
 * @description TODO
 * @Version 1.0
 **/
@Configuration
@ConfigurationProperties(prefix = "spring.kafka.consumer")
public class KafkaConsumerConfig {

    /** group-id **/
    public static String groupId="group-1";

    /** topic name **/
    public static String topic ;

    /** 当Kafka中没有初始偏移量或者服务器上不再存在当前偏移量时该怎么办，默认值为latest，表示自动将偏移重置为最新的偏移量
     * 可选的值为latest, earliest, none
     **/
    public static String autoOffsetReset;

    /** key序列化方式 **/
    public static String keyDeserializer;

    /** value序列化方式 **/
    public static String valueDeserializer ;

    /**  **/
    public static String isolationLevel;

    /**  **/
    public static String fetchMinSize;

    /** 每次拉取最大的数据 **/
    public static String maxPollRecords;

    /** 如果'enable.auto.commit'为true，则消费者偏移自动提交给Kafka的频率（以毫秒为单位），默认值为5000 **/
    public static int autoCommitInterval;

    /** 如果为true，则消费者的偏移量将在后台定期提交，默认值为true **/
    public static boolean enableAutoCommit;

    public static String getGroupId() {
        return groupId;
    }

    public void setGroupId(String groupId) {
        KafkaConsumerConfig.groupId = groupId;
    }

    public static String getTopic() {
        return topic;
    }

    public void setTopic(String topic) {
        KafkaConsumerConfig.topic = topic;
    }

    public static String getAutoOffsetReset() {
        return autoOffsetReset;
    }

    public void setAutoOffsetReset(String autoOffsetReset) {
        KafkaConsumerConfig.autoOffsetReset = autoOffsetReset;
    }

    public static String getKeyDeserializer() {
        return keyDeserializer;
    }

    public void setKeyDeserializer(String keyDeserializer) {
        KafkaConsumerConfig.keyDeserializer = keyDeserializer;
    }

    public static String getValueDeserializer() {
        return valueDeserializer;
    }

    public void setValueDeserializer(String valueDeserializer) {
        KafkaConsumerConfig.valueDeserializer = valueDeserializer;
    }

    public static String getIsolationLevel() {
        return isolationLevel;
    }

    public void setIsolationLevel(String isolationLevel) {
        KafkaConsumerConfig.isolationLevel = isolationLevel;
    }

    public static String getFetchMinSize() {
        return fetchMinSize;
    }

    public void setFetchMinSize(String fetchMinSize) {
        KafkaConsumerConfig.fetchMinSize = fetchMinSize;
    }

    public static String getMaxPollRecords() {
        return maxPollRecords;
    }

    public void setMaxPollRecords(String maxPollRecords) {
        KafkaConsumerConfig.maxPollRecords = maxPollRecords;
    }

    public static int getAutoCommitInterval() {
        return autoCommitInterval;
    }

    public void setAutoCommitInterval(int autoCommitInterval) {
        KafkaConsumerConfig.autoCommitInterval = autoCommitInterval;
    }

    public static boolean getEnableAutoCommit() {
        return enableAutoCommit;
    }

    public void setEnableAutoCommit(boolean enableAutoCommit) {
        KafkaConsumerConfig.enableAutoCommit = enableAutoCommit;
    }
}

package com.hnu.common.kafka.config;

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;

/**
 * @author yuanhx
 * @Date 2021/6/3
 * @description TODO
 * @Version 1.0
 **/
@Configuration
@ConfigurationProperties(prefix = "spring.kafka")
public class KafkaServerConfig {

    /** kafka消费者服务器地址 **/

    private static String bootstrapServers;

    public static String getBootstrapServers() {
        return bootstrapServers;
    }

    public  void setBootstrapServers(String bootstrapServers) {
        KafkaServerConfig.bootstrapServers = bootstrapServers;
    }
}

package com.hnu.common.kafka.config;

import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;

@Component
public class SpringContextUtil implements ApplicationContextAware {
	private static ApplicationContext applicationContext;

	@Override
	public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
		SpringContextUtil.applicationConte