kafka消费者源码分析

01 KafKaConsumer实例化

KafkaConsumer(ConsumerConfig config, Deserializer<K> keyDeserializer, Deserializer<V> valueDeserializer) {
    this.closed = false;
    this.currentThread = new AtomicLong(-1L);
    this.refcount = new AtomicInteger(0);

    try {
        GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, ProtocolType.CONSUMER);
        // 消费组id
        this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId);
        // 获取客户端id
        this.clientId = config.getString("client.id");
        LogContext logContext;
        if (groupRebalanceConfig.groupInstanceId.isPresent()) {
            logContext = new LogContext("[Consumer instanceId=" + (String)groupRebalanceConfig.groupInstanceId.get() + ", clientId=" + this.clientId + ", groupId=" + (String)this.groupId.orElse("null") + "] ");
        } else {
            logContext = new LogContext("[Consumer clientId=" + this.clientId + ", groupId=" + (String)this.groupId.orElse("null") + "] ");
        }

        this.log = logContext.logger(this.getClass());
        boolean enableAutoCommit = config.maybeOverrideEnableAutoCommit();
        this.groupId.ifPresent((groupIdStr) -> {
            if (groupIdStr.isEmpty()) {
                this.log.warn("Support for using the empty group id by consumers is deprecated and will be removed in the next major release.");
            }

        });
        this.log.debug("Initializing the Kafka consumer");
        this.requestTimeoutMs = (long)config.getInt("request.timeout.ms");
        this.defaultApiTimeoutMs = config.getInt("default.api.timeout.ms");
        this.time = Time.SYSTEM;
        this.metrics = buildMetrics(config, this.time, this.clientId);
        this.retryBackoffMs = config.getLong("retry.backoff.ms");
        // 拦截器
        List<ConsumerInterceptor<K, V>> interceptorList = config.getConfiguredInstances("interceptor.classes", ConsumerInterceptor.class, Collections.singletonMap("client.id", this.clientId));
        this.interceptors = new ConsumerInterceptors(interceptorList);
        // key的反序列化
        if (keyDeserializer == null) {
            this.keyDeserializer = (Deserializer)config.getConfiguredInstance("key.deserializer", Deserializer.class);
            this.keyDeserializer.configure(config.originals(Collections.singletonMap("client.id", this.clientId)), true);
        } else {
            config.ignore("key.deserializer");
            this.keyDeserializer = keyDeserializer;
        }
        // value的反序列化
        if (valueDeserializer == null) {
            this.valueDeserializer = (Deserializer)config.getConfiguredInstance("value.deserializer", Deserializer.class);
            this.valueDeserializer.configure(config.originals(Collections.singletonMap("client.id", this.clientId)), false);
        } else {
            config.ignore("value.deserializer");
            this.valueDeserializer = valueDeserializer;
        }

        OffsetResetStrategy offsetResetStrategy = OffsetResetStrategy.valueOf(config.getString("auto.offset.reset").toUpperCase(Locale.ROOT));
        // 创建订阅的对象,用户封装订阅信息
        this.subscriptions = new SubscriptionState(logContext, offsetResetStrategy);
        ClusterResourceListeners clusterResourceListeners = this.configureClusterResourceListeners(keyDeserializer, valueDeserializer, this.metrics.reporters(), interceptorList);
        // 更新集群元数据
        this.metadata = new ConsumerMetadata(this.retryBackoffMs, config.getLong("metadata.max.age.ms"), !config.getBoolean("exclude.internal.topics"), config.getBoolean("allow.auto.create.topics"), this.subscriptions, logContext, clusterResourceListeners);
        List<InetSocketAddress> addresses = ClientUtils.parseAndValidateAddresses(config.getList("bootstrap.servers"), config.getString("client.dns.lookup"));
        this.metadata.bootstrap(addresses);
        String metricGrpPrefix = "consumer";
        FetcherMetricsRegistry metricsRegistry = new FetcherMetricsRegistry(Collections.singleton("client-id"), metricGrpPrefix);
        // 创建channel的构建器
        ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config, this.time, logContext);
        // 设置隔离级别
        IsolationLevel isolationLevel = IsolationLevel.valueOf(config.getString("isolation.level").toUpperCase(Locale.ROOT));
        Sensor throttleTimeSensor = Fetcher.throttleTimeSensor(this.metrics, metricsRegistry);
        int heartbeatIntervalMs = config.getInt("heartbeat.interval.ms");
        ApiVersions apiVersions = new ApiVersions();
        // 实例化网络客户端
        NetworkClient netClient = new NetworkClient(new Selector(config.getLong("connections.max.idle.ms"), this.metrics, this.time, metricGrpPrefix, channelBuilder, logContext), this.metadata, this.clientId, 100, config.getLong("reconnect.backoff.ms"), config.getLong("reconnect.backoff.max.ms"), config.getInt("send.buffer.bytes"), config.getInt("receive.buffer.bytes"), config.getInt("request.timeout.ms"), config.getLong("socket.connection.setup.timeout.ms"), config.getLong("socket.connection.setup.timeout.max.ms"), ClientDnsLookup.forConfig(config.getString("client.dns.lookup")), this.time, true, apiVersions, throttleTimeSensor, logContext);
        // 实例化消费者网络客户端
        this.client = new ConsumerNetworkClient(logContext, netClient, this.metadata, this.time, this.retryBackoffMs, config.getInt("request.timeout.ms"), heartbeatIntervalMs);
        // 消费组和主题分区分配器
        this.assignors = PartitionAssignorAdapter.getAssignorInstances(config.getList("partition.assignment.strategy"), config.originals(Collections.singletonMap("client.id", this.clientId)));
        // offset协调者
        this.coordinator = !this.groupId.isPresent() ? null : new ConsumerCoordinator(groupRebalanceConfig, logContext, this.client, this.assignors, this.metadata, this.subscriptions, this.metrics, metricGrpPrefix, this.time, enableAutoCommit, config.getInt("auto.commit.interval.ms"), this.interceptors, config.getBoolean("internal.throw.on.fetch.stable.offset.unsupported"));
        // 用于通过网络获取消息的对象
        this.fetcher = new Fetcher(logContext, this.client, config.getInt("fetch.min.bytes"), config.getInt("fetch.max.bytes"), config.getInt("fetch.max.wait.ms"), config.getInt("max.partition.fetch.bytes"), config.getInt("max.poll.records"), config.getBoolean("check.crcs"), config.getString("client.rack"), this.keyDeserializer, this.valueDeserializer, this.metadata, this.subscriptions, this.metrics, metricsRegistry, this.time, this.retryBackoffMs, this.requestTimeoutMs, isolationLevel, apiVersions);
        this.kafkaConsumerMetrics = new KafkaConsumerMetrics(this.metrics, metricGrpPrefix);
        config.logUnused();
        AppInfoParser.registerAppInfo("kafka.consumer", this.clientId, this.metrics, this.time.milliseconds());
        this.log.debug("Kafka consumer initialized");
    } catch (Throwable var19) {
        if (this.log != null) {
            this.close(0L, true);
        }

        throw new KafkaException("Failed to construct kafka consumer", var19);
    }
}

---

02 订阅主题

public void subscribe(Collection<String> topics, ConsumerRebalanceListener listener) {
    // 轻量级锁
    this.acquireAndEnsureOpen();

    try {
        this.maybeThrowInvalidGroupIdException();
        if (topics == null) {
            throw new IllegalArgumentException("Topic collection to subscribe to cannot be null");
        }

        if (topics.isEmpty()) {
            // 如果topic为空,就不订阅
            this.unsubscribe();
        } else {
            Iterator var3 = topics.iterator();

            String topic;
            do {
                if (!var3.hasNext()) {
                    this.throwIfNoAssignorsConfigured();
                    this.fetcher.clearBufferedDataForUnassignedTopics(topics);
                    this.log.info("Subscribed to topic(s): {}", Utils.join(topics, ", "));
                    // 在这一步订阅
                    if (this.subscriptions.subscribe(new HashSet(topics), listener)) {
                        // 更新元数据,如果metadata当前不包括所有的topics,则标记强制更新
                        this.metadata.requestUpdateForNewTopics();
                    }

                    return;
                }

                topic = (String)var3.next();
            } while(topic != null && !topic.trim().isEmpty());

            throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic");
        }
    } finally {
        this.release();
    }

}

进入这一步 this.subscriptions.subscribe(new HashSet(topics), listener))

public synchronized boolean subscribe(Set<String> topics, ConsumerRebalanceListener listener) {
    // 监听
    this.registerRebalanceListener(listener);
    // 按照指定topic进行订阅,自动分配分区
    this.setSubscriptionType(SubscriptionState.SubscriptionType.AUTO_TOPICS);
    // 修改订阅信息
    return this.changeSubscription(topics);
}
进入changeSubscription(topics)
private boolean changeSubscription(Set<String> topicsToSubscribe) {
    if (this.subscription.equals(topicsToSubscribe)) {
        return false;
    } else {
        this.subscription = topicsToSubscribe;
        return true;
    }
}

---

消费消息poll()

private ConsumerRecords<K, V> poll(Timer timer, boolean includeMetadataInTimeout) {
    this.acquireAndEnsureOpen();  // 使用轻量级锁检测kafkaConsumer是否被其他线程使用

    try {
        this.kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs());
        // 订阅类型为NONE抛异常,表示当前消费者没有订阅任何topic或者没有分配分区
        if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) {
            throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions");
        } else {
            do {
                this.client.maybeTriggerWakeup();
                if (includeMetadataInTimeout) {
                    this.updateAssignmentMetadataIfNeeded(timer, false);
                } else {
                    while(!this.updateAssignmentMetadataIfNeeded(this.time.timer(9223372036854775807L), true)) {
                        this.log.warn("Still waiting for metadata");
                    }
                }
                    // 核心方法，拉取消息
                Map<TopicPartition, List<ConsumerRecord<K, V>>> records = this.pollForFetches(timer);
                if (!records.isEmpty()) {
                    if (this.fetcher.sendFetches() > 0 || this.client.hasPendingRequests()) {
                        this.client.transmitSends();
                    }
                    // 拦截器        
                    ConsumerRecords var4 = this.interceptors.onConsume(new ConsumerRecords(records));
                    return var4;
                }
            } while(timer.notExpired());

            ConsumerRecords var8 = ConsumerRecords.empty();
            return var8;
        }
    } finally {
        this.release();
        this.kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs());
    }
}