kafka 消费者-comsumer

public class Consumer extends ShutdownableThread {
    private final KafkaConsumer<Integer, String> consumer;
    private final String topic;

    public Consumer(String topic) {
        super("KafkaConsumerExample", false);
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "DemoConsumer");
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");

        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.IntegerDeserializer");
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        //consumer里面其实就是初始化了几个核心的组件 NetworkClient ConsumerCoordinator Fetcher。
        consumer = new KafkaConsumer<>(props);
        this.topic = topic;
    }

    @Override
    public void doWork() {
        //订阅了主题
        consumer.subscribe(Collections.singletonList(this.topic));
        //TODO 里面会有重要代码
        ConsumerRecords<Integer, String> records = consumer.poll(1000);
        for (ConsumerRecord<Integer, String> record : records) {
            System.out.println("Received message: (" + record.key() + ", " + record.value() + ") at offset " + record.offset());
        }
    }
}
-> consumer.poll(1000) -> pollOnce
    private Map<TopicPartition, List<ConsumerRecord<K, V>>> pollOnce(long timeout) {
        // TODO
        coordinator.poll(time.milliseconds());

        // fetch positions if we have partitions we're subscribed to that we
        // don't know the offset for
        if (!subscriptions.hasAllFetchPositions())
            updateFetchPositions(this.subscriptions.missingFetchPositions());

        // if data is available already, return it immediately
        Map<TopicPartition, List<ConsumerRecord<K, V>>> records = fetcher.fetchedRecords();
        if (!records.isEmpty())
            return records;

        // send any new fetches (won't resend pending fetches)
        //TODO 发送网络请求
        fetcher.sendFetches();

        long now = time.milliseconds();
        long pollTimeout = Math.min(coordinator.timeToNextPoll(now), timeout);

        client.poll(pollTimeout, now, new PollCondition() {
            @Override
            public boolean shouldBlock() {
                // since a fetch might be completed by the background thread, we need this poll condition
                // to ensure that we do not block unnecessarily in poll()
                return !fetcher.hasCompletedFetches();
            }
        });

        // after the long poll, we should check whether the group needs to rebalance
        // prior to returning data so that the group can stabilize faster
        if (coordinator.needRejoin())
            return Collections.emptyMap();
        //获取到请求的结果。
        return fetcher.fetchedRecords();
    }
-> coordinator.poll(time.milliseconds()); 
//计算出来哪台服务器是coondinator服务器了
-> ensureCoordinatorReady -> lookupCoordinator
   //TODO 发送请求
   findCoordinatorFuture = sendGroupCoordinatorRequest(node);
        //发送的是GROUP_COORDINATOR的请求  Kafkaapis
        return client.send(node, ApiKeys.GROUP_COORDINATOR, metadataRequest)
                        //TODO 对响应进行处理
                     .compose(new GroupCoordinatorResponseHandler());

-> GroupCoordinatorResponseHandler(对响应进行处理)
           synchronized (AbstractCoordinator.this) {
                    //封装出来一台服务器，coordinator服务器
                    AbstractCoordinator.this.coordinator = new Node(
                            //从响应里面解析出来 coordinator服务器的信息
                            Integer.MAX_VALUE - groupCoordinatorResponse.node().id(),
                            groupCoordinatorResponse.node().host(),
                            groupCoordinatorResponse.node().port());
                    log.info("Discovered coordinator {} for group {}.", coordinator, groupId);
                    client.tryConnect(coordinator);
                    heartbeat.resetTimeouts(time.milliseconds());
                }

如何选举consumer leader (谁先注册上来，谁就是leader consumer)

ConsumerCondinator.poll.ensureActiveGroup(); -> joinGroupIfNeeded
  RequestFuture<ByteBuffer> future = initiateJoinGroup();
     //TODO 发送一个注册的请求
     joinFuture = sendJoinGroupRequest();
     //在发送请求   ApiKeys.JOIN_GROUP
        return client.send(coordinator, ApiKeys.JOIN_GROUP, request)
                .compose(new JoinGroupResponseHandler());

-> KafkaApi.case ApiKeys.JOIN_GROUP => handleJoinGroupRequest(request)
coordinator.handleJoinGroup( -> doJoinGroup -> addMemberAndRebalance
    //TODO 添加自己的信息
    group.add(member.memberId, member)
  ->   
  def add(memberId: String, member: MemberMetadata) {
    if (members.isEmpty)
      this.protocolType = Some(member.protocolType)

    assert(groupId == member.groupId)
    assert(this.protocolType.orNull == member.protocolType)
    assert(supportsProtocols(member.protocols))
    //第一次进来的时候leaderid可定是null的
    if (leaderId == null)
      //我们就知道了，其实让哪个consumer作为leader consumer很简单
      //就是谁先注册上来，谁就是leader consumer。
      leaderId = memberId
    members.put(memberId, member)
  }

leader 如何制定分区方案

接上篇
        //在发送请求  ApiKeys.JOIN_GROUP
        return client.send(coordinator, ApiKeys.JOIN_GROUP, request)
                .compose(new JoinGroupResponseHandler());

-> JoinGroupResponseHandler
       //TODO 我们所有的消费组里面的成员
       //都会去发送joinGroup
       //最终只会有一个consumer是leader
       //如果发现自己是leader的consumer，那么就会调用
       //这个方法，然后制定分区方案
       //把分区方案发送给coondinator
       onJoinLeader(joinResponse).chain(future);
       
    private RequestFuture<ByteBuffer> onJoinLeader(JoinGroupResponse joinResponse) {
        try {
            // perform the leader synchronization and send back the assignment for the group
            //TODO 这个方法就是指定分区方案的方法
            Map<String, ByteBuffer> groupAssignment = performAssignment(joinResponse.leaderId(), joinResponse.groupProtocol(),
                    joinResponse.members());

            SyncGroupRequest request = new SyncGroupRequest(groupId, generation.generationId, generation.memberId, groupAssignment);
            log.debug("Sending leader SyncGroup for group {} to coordinator {}: {}", groupId, this.coordinator, request);
            //TODO 发送请求
            return sendSyncGroupRequest(request);
        } catch (RuntimeException e) {
            return RequestFuture.failure(e);
        }
    }

sendSyncGroupRequest
        //发送SYNC_GROUP请求
        return client.send(coordinator, ApiKeys.SYNC_GROUP, request)
                .compose(new SyncGroupResponseHandler());

coodinator 下发分区方案

接上篇
        //发送SYNC_GROUP请求
        return client.send(coordinator, ApiKeys.SYNC_GROUP, request)
                .compose(new SyncGroupResponseHandler());
-> KafkaApi. ApiKeys.SYNC_GROUP => handleSyncGroupRequest(request)
 coordinator.handleSyncGroup( -> doSyncGroup
 //TODO coondinator下发分区方案
  setAndPropagateAssignment(group, assignment)
      propagateAssignment
  ->  private def propagateAssignment(group: GroupMetadata, error: Errors) {
    //遍历所有的member
    for (member <- group.allMemberMetadata) {
      if (member.awaitingSyncCallback != null) {
        //调用回调函数
        //这个assignment参数就是分配的分区消费方案。
        //其实coondinator就是通过调用这个member的回调函数，然后完成的分区方案的下发。
        member.awaitingSyncCallback(member.assignment, error.code)
        member.awaitingSyncCallback = null

        // reset the session timeout for members after propagating the member's assignment.
        // This is because if any member's session expired while we were still awaiting either
        // the leader sync group or the storage callback, its expiration will be ignored and no
        // future heartbeat expectations will not be scheduled.
        completeAndScheduleNextHeartbeatExpiration(group, member)
      }
    }
  }

consumer消费数据

KafkaConsumer.pollOnce
fetcher.sendFetches();
        // TODO ApiKeys.FETCH
        client.send(fetchTarget, ApiKeys.FETCH, request)
//获取到请求的结果。
 return fetcher.fetchedRecords();

comsumer 提交偏移量

    //配置是否进行自动提交偏移的标志位
    private final boolean autoCommitEnabled;
    //每隔多久提交一次偏移量信息（消费offset）
    private final int autoCommitIntervalMs;
ConsumerCoordinator.commitOffsetsAsync
    public void commitOffsetsAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, final OffsetCommitCallback callback) {
        invokeCompletedOffsetCommitCallbacks();
        //其实提交偏移量信息就是要提交
        //coonrdinator
        //offset ->  __consumer_offset 默认有50个分区 -》 4 leader partition 在哪台主机
        //那么哪一台就是coondinator
        //同时，我们这个消费组的 偏移量信息也是提交到这一台服务器（partition4这个leader partition）
        //
        if (!coordinatorUnknown()) {
            //TODO
            doCommitOffsetsAsync(offsets, callback);
            ...
            RequestFuture<Void> future = sendOffsetCommitRequest(offsets);
            //发送请求  ApiKeys.OFFSET_COMMIT
              return client.send(coordinator, ApiKeys.OFFSET_COMMIT, req)
                .compose(new OffsetCommitResponseHandler(offsets));

consumer 和 coordinator心跳

AbstractCoordinator.HeartbeatThread.run
     //设置心跳的时间
      heartbeat.sentHeartbeat(now);
       //发送心跳的请求
       sendHeartbeatRequest().addListener(new RequestFutureListener<Void>() 
        //把请求发送出去
        return client.send(coordinator, ApiKeys.HEARTBEAT, req)
                .compose(new HeartbeatResponseHandler());

KafkaApi.case ApiKeys.HEARTBEAT => handleHeartbeatRequest(request)
coordinator.handleHeartbeat(  -> completeAndScheduleNextHeartbeatExpiration
  private def completeAndScheduleNextHeartbeatExpiration(group: GroupMetadata, member: MemberMetadata) {
    // complete current heartbeat expectation
    // 更新对应的consumer的上一次的心跳时间
    member.latestHeartbeat = time.milliseconds()
    val memberKey = MemberKey(member.groupId, member.memberId)
    heartbeatPurgatory.checkAndComplete(memberKey)

    // reschedule the next heartbeat expiration deadline
    //时间轮机制，把往时间轮里面插入一个任务，这个任务就是用来检查心跳是否超时的。
    //11:00：00 心跳  11:00：30
    //去看最后30秒是否有心跳超时的。

    //作为一个老师，给大家留一点空间，作业。
    //自己下去，根据老师之前怎么带大家去分析 Producer server源码
    //consumer的详细的流程。

    val newHeartbeatDeadline = member.latestHeartbeat + member.sessionTimeoutMs
    val delayedHeartbeat = new DelayedHeartbeat(this, group, member, newHeartbeatDeadline, member.sessionTimeoutMs)
    heartbeatPurgatory.tryCompleteElseWatch(delayedHeartbeat, Seq(memberKey))
  }