正文
系列目录
kafka原理和实践（三）spring-kafka生产者源码
kafka原理和实践（四）spring-kafka消费者源码
==============正文分割线=====================
一、kafkaConsumer消费者模型
如上图所示，spring-kafka消费者模型主要流程：
1.容器启动，轮询执行消费。
2.kafkaConsumer拉取消息流程：
1）Fetcher请求获取器获取请求并存储在unset中
2）ConsumerNetworkClient网络客户端执行poll(),调用NetWlrikClient的send()方法从unset中获取ClientRequest请求转成RequestSend最终塞进Selector的KafkaChannel通道中，Seletcor.send()从kafka集群拉取待消费数据ConsumerRecords
3. 消费者监听器MessageListener.onMessage()执行用户自定义的实际消费业务逻辑。
一、kafkaConsumer构造
1 @SuppressWarnings("unchecked") 2 private KafkaConsumer(ConsumerConfig config, 3 Deserializer<K> keyDeserializer, 4 Deserializer<V> valueDeserializer) { 5 try { 6 log.debug("Starting the Kafka consumer"); 7 this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); 8 int sessionTimeOutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); 9 int fetchMaxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); 10 if (this.requestTimeoutMs <= sessionTimeOutMs || this.requestTimeoutMs <= fetchMaxWaitMs) 11 throw new ConfigException(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " should be greater than " + ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG + " and " + ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); 12 this.time = new SystemTime(); 13 14 String clientId = config.getString(ConsumerConfig.CLIENT_ID_CONFIG); 15 if (clientId.length() <= 0) 16 clientId = "consumer-" + CONSUMER_CLIENT_ID_SEQUENCE.getAndIncrement(); 17 this.clientId = clientId; 18 Map<String, String> metricsTags = new LinkedHashMap<>(); 19 metricsTags.put("client-id", clientId); 20 MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG)) 21 .timeWindow(config.getLong(ConsumerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) 22 .tags(metricsTags); 23 List<MetricsReporter> reporters = config.getConfiguredInstances(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, 24 MetricsReporter.class); 25 reporters.add(new JmxReporter(JMX_PREFIX)); 26 this.metrics = new Metrics(metricConfig, reporters, time); 27 this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); 28 29 // load interceptors and make sure they get clientId 30 Map<String, Object> userProvidedConfigs = config.originals(); 31 userProvidedConfigs.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); 32 List<ConsumerInterceptor<K, V>> interceptorList = (List) (new ConsumerConfig(userProvidedConfigs)).getConfiguredInstances(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, 33 ConsumerInterceptor.class); 34 this.interceptors = interceptorList.isEmpty() ? null : new ConsumerInterceptors<>(interceptorList); 35 if (keyDeserializer == null) { 36 this.keyDeserializer = config.getConfiguredInstance(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, 37 Deserializer.class); 38 this.keyDeserializer.configure(config.originals(), true); 39 } else { 40 config.ignore(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG); 41 this.keyDeserializer = keyDeserializer; 42 } 43 if (valueDeserializer == null) { 44 this.valueDeserializer = config.getConfiguredInstance(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, 45 Deserializer.class); 46 this.valueDeserializer.configure(config.originals(), false); 47 } else { 48 config.ignore(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG); 49 this.valueDeserializer = valueDeserializer; 50 } 51 ClusterResourceListeners clusterResourceListeners = configureClusterResourceListeners(keyDeserializer, valueDeserializer, reporters, interceptorList); 52 this.metadata = new Metadata(retryBackoffMs, config.getLong(ConsumerConfig.METADATA_MAX_AGE_CONFIG), false, clusterResourceListeners); 53 List<InetSocketAddress> addresses = ClientUtils.parseAndValidateAddresses(config.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)); 54 this.metadata.update(Cluster.bootstrap(addresses), 0); 55 String metricGrpPrefix = "consumer"; 56 ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config.values()); 57 NetworkClient netClient = new NetworkClient( 58 new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder), 59 this.metadata, 60 clientId, 61 100, // a fixed large enough value will suffice 62 config.getLong(ConsumerConfig.RECONNECT_BACKOFF_MS_CONFIG), 63 config.getInt(ConsumerConfig.SEND_BUFFER_CONFIG), 64 config.getInt(ConsumerConfig.RECEIVE_BUFFER_CONFIG), 65 config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG), time); 66 this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs, 67 config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG)); 68 OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toUpperCase(Locale.ROOT)); 69 this.subscriptions = new SubscriptionState(offsetResetStrategy); 70 List<PartitionAssignor> assignors = config.getConfiguredInstances( 71 ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG, 72 PartitionAssignor.class); 73 this.coordinator = new ConsumerCoordinator(this.client, 74 config.getString(ConsumerConfig.GROUP_ID_CONFIG), 75 config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), 76 config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), 77 config.getInt(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG), 78 assignors, 79 this.metadata, 80 this.subscriptions, 81 metrics, 82 metricGrpPrefix, 83 this.time, 84 retryBackoffMs, 85 new ConsumerCoordinator.DefaultOffsetCommitCallback(), 86 config.getBoolean(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), 87 config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), 88 this.interceptors, 89 config.getBoolean(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG)); 90 this.fetcher = new Fetcher<>(this.client, 91 config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG), 92 config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG), 93 config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG), 94 config.getInt(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG), 95 config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), 96 config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG), 97 this.keyDeserializer, 98 this.valueDeserializer, 99 this.metadata, 100 this.subscriptions, 101 metrics, 102 metricGrpPrefix, 103 this.time, 104 this.retryBackoffMs); 105 106 config.logUnused(); 107 AppInfoParser.registerAppInfo(JMX_PREFIX, clientId); 108 109 log.debug("Kafka consumer created"); 110 } catch (Throwable t) { 111 // call close methods if internal objects are already constructed 112 // this is to prevent resource leak. see KAFKA-2121 113 close(true); 114 // now propagate the exception 115 throw new KafkaException("Failed to construct kafka consumer", t); 116 } 117 }
从KafkaConsumer构造函数来看，核心组件有：
1.Metadata：封装了元数据的一些逻辑的类。元数据仅保留一个主题的子集，随着时间的推移可以添加。当我们请求一个主题的元数据时，我们没有任何元数据会触发元数据更新。如果对元数据启用了主题过期，那么在更新之后，在过期时间间隔内未使用的任何主题都将从元数据刷新集中删除。
2.ConsumerNetworkClient：高等级消费者访问网络层，为请求Future任务提供基本支持。这个类是线程安全的，但是不提供响应回调的同步。这保证在调用它们时不会持有锁。
3.SubscriptionState：订阅的TopicPartition的offset状态维护
4.ConsumerCoordinator：消费者的协调者，负责partitiion的分配，reblance
5.Fetcher：从brokers上按照配置获取消息。
二、消费者容器启动流程
kafka消费者有两种常见的实现方式：
1.xml配置文件
2.基于注解实现
其实，不管哪种方式，本质只是生成Spring Bean的方式不同而已。我们就以xml的实现方式来追踪源码。
基于xml的总体配置如下：
1 <!-- 1.定义consumer的参数 -->
2 <bean id="consumerProperties" class="java.util.HashMap">
3 <constructor-arg>
4 <map> 5 <entry key="bootstrap.servers" value="${bootstrap.servers}" /> 6 <entry key="group.id" value="${group.id}" /> 7 <entry key="enable.auto.commit" value="${enable.auto.commit}" /> 8 <entry key="session.timeout.ms" value="${session.timeout.ms}" /> 9 <entry key="key.deserializer" 10 value="org.apache.kafka.common.serialization.StringDeserializer" /> 11 <entry key="value.deserializer" 12 value="org.apache.kafka.common.serialization.StringDeserializer" /> 13 </map> 14 </constructor-arg> 15 </bean> 16 17 <!-- 2.创建consumerFactory bean --> 18 <bean id="consumerFactory" 19 class="org.springframework.kafka.core.DefaultKafkaConsumerFactory" > 20 <constructor-arg> 21 <ref bean="consumerProperties" /> 22 </constructor-arg> 23 </bean> 24 25 <!-- 3.定义消费实现类 --> 26 <bean id="kafkaConsumerService" class="xxx.service.impl.KafkaConsumerSerivceImpl" /> 27 28 <!-- 4.消费者容器配置信息 --> 29 <bean id="containerProperties" class="org.springframework.kafka.listener.config.ContainerProperties"> 30 <!-- topic --> 31 <constructor-arg name="topics"> 32 <list> 33 <value>${kafka.consumer.topic.credit.for.lease}</value> 34 <value>${loan.application.feedback.topic}</value> 35 <value>${templar.agreement.feedback.topic}</value> 36 <value>${templar.aggrement.active.feedback.topic}</value> 37 <value>${templar.aggrement.agreementRepaid.topic}</value> 38 <value>${templar.aggrement.agreementWithhold.topic}</value> 39 <value>${templar.aggrement.agreementRepayRemind.topic}</value> 40 </list> 41 </constructor-arg> 42 <property name="messageListener" ref="kafkaConsumerService" /> 43 </bean> 44 <!-- 5.消费者并发消
最低0.47元/天 解锁文章
4970
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
