ConsumerNetworkClient 分析

ConsumerNetworkClient 是对NetworkClient的一个封装，提供了额外的一些功能，首先我们看一下它有哪些属性：

# NetworkClient

private final KafkaClient client;

 

# 等待被发送的请求队列

private final Map<Node,List<ClientRequest>> unsent = new HashMap<>();

 

# 元数据信息

private final Metadata metadata;

# 重试时间

private final long retryBackoffMs;

# ClientRequest在unsent列表的超时时间

private final long unsentExpiryMs;

 

# KafkaConsumer是否正在执行不可中断的方法的次数，每次进入一个不可中断方法就加1，退出减1，它只会被KafkaConsumer线程修改，其他线程不能修改

private int wakeupDisabledCount = 0;

 

# 当请求完成时，它们会在调用之前被转移到这个队列中。这样做的目的是为了避免在持有该对象的监视器时调用它们，从而为死锁打开大门。

private finalConcurrentLinkedQueue<RequestFutureCompletionHandler> pendingCompletion =new ConcurrentLinkedQueue<>();

 

接着我们看一些它的核心方法：

# poll(long timeout, long now, PollConditionpollCondition) 主要用于发送请求，检测连接状态，处理超时请求

public void poll(long timeout, long now, PollCondition pollCondition) {
    // 触发pendingCompleted里的请求
    firePendingCompletedRequests();

    synchronized (this) {
        // 发送现在我们能发送的所有请求
        trySend(now);

        // 检测我们是否还需要poll
        if (pollCondition == null || pollCondition.shouldBlock()) {
            // if thereare no requests in flight, do not block longer than the retry backoff
            if (client.inFlightRequestCount() == 0)
                timeout = Math.min(timeout, retryBackoffMs);
            client.poll(Math.min(MAX_POLL_TIMEOUT_MS, timeout), now);
            now = time.milliseconds();
        } else {
            client.poll(0, now);
        }
        // 检测连接状态，检测消费者与每个Node之间的连接状态，当检测到连接断开的Node时，会将其在unsent列表中对应的全部Client
        // Request对象清除掉，之后调用这些ClientRequest函数，并且设置disconnect标记为true
        checkDisconnects(now);
       
        // 检测wakeup 和wakeupDisabledCount,查看是否有其他线程中断，如果有中断请求，则抛出WakeupExeception
        maybeTriggerWakeup();
        // 再次试图发送，可能现在内存已经清理了或者Node可以连接上了
        trySend(now);

        // 处理unsent中的超时请求，它会遍历整个unsent集合，检测，每一个ClientRequest是否超时，调用超时ClientRequest
        // 的回调函数，并将其从unsent列表删除
        failExpiredRequests(now);
    }

    // 触发pendingCompleted里的请求
    firePendingCompletedRequests();
}

 

# poll(RequestFuture<?> future, longtimeout): 是一个实现阻塞发送请求的功能

public boolean poll(RequestFuture<?> future, long timeout) {
    long begin = time.milliseconds();
    long remaining = timeout;
    long now = begin;
    do {
        poll(remaining, now, future);// 请求未完成则调用poll方法
        now = time.milliseconds();
        long elapsed = now - begin;
        remaining = timeout - elapsed;
    } while (!future.isDone() && remaining > 0); //循环检测future的完成情况
    return future.isDone();
}

 

# send(Node, ApiKeys api, shortversion,AbstractRequest request):

会将等待发送的请求封装成ClinetRequest

private RequestFuture<ClientResponse> send(Node node, ApiKeys api, short version, AbstractRequest request) {
    long now = time.milliseconds();
    RequestFutureCompletionHandler completionHandler = new RequestFutureCompletionHandler();
    RequestHeader header = client.nextRequestHeader(api, version);
    RequestSend send = new RequestSend(node.idString(), header, request.toStruct());
    // 将这个ClinetRequest放到unsent列表
    put(node, new ClientRequest(now, true, send, completionHandler));

    // 如果NetworkCLient被poll阻塞，则唤醒它
    client.wakeup();
    // 返回RequestFuture对象
    return completionHandler.future;
}

 

# put(Node node, ClientRequest request): 往unsent列表添加ClientRequest 这些请求等待被发送

private void put(Node node, ClientRequest request) {
    synchronized (this) {
        List<ClientRequest> nodeUnsent = unsent.get(node);
        if (nodeUnsent == null) {
            nodeUnsent = new ArrayList<>();
            unsent.put(node, nodeUnsent);
        }
        nodeUnsent.add(request);
    }
}

 

# leastLoadedNode: 获取集群中负载最少的节点

public Node leastLoadedNode() {
    synchronized (this) {
        return client.leastLoadedNode(time.milliseconds());
    }
}

 

# awaitMetadataUpdate(): 除非元数据已经更新，否则一直阻塞

public void awaitMetadataUpdate() {
    awaitMetadataUpdate(Long.MAX_VALUE);
}

 

# awaitMetadataUpdate(long timeout): 除非元数据已经更新，否则在指定的时间内一直阻塞

public boolean awaitMetadataUpdate(long timeout) {
    long startMs = time.milliseconds();
    int version = this.metadata.requestUpdate();
    do {
        poll(timeout);
    } while (this.metadata.version() == version && time.milliseconds() - startMs < timeout);
    return this.metadata.version() > version;
}

 

# ensureFreshMetadata(): 确保元数据应经刷新

public void ensureFreshMetadata() {
    if (this.metadata.updateRequested() || this.metadata.timeToNextUpdate(time.milliseconds()) == 0)
        awaitMetadataUpdate();
}

 

# awaitPendingRequests(Node node): 等待指定节点的pedning的请求已经完成，否则一直阻塞

public void awaitPendingRequests(Node node) {
    while (pendingRequestCount(node) > 0)
        poll(retryBackoffMs);
}

 

# pendingRequestCount(Node node)： 获得指定节点pending的request数量

public int pendingRequestCount(Node node) {
    synchronized (this) {
        List<ClientRequest> pending = unsent.get(node);
        int unsentCount = pending == null ? 0 : pending.size();
        return unsentCount + client.inFlightRequestCount(node.idString());
    }
}

 

# pendingRequestCount()： 获得所有节点pending的request数量

public int pendingRequestCount() {
    synchronized (this) {
        int total = 0;
        for (List<ClientRequest> requests: unsent.values())
            total += requests.size();
        return total + client.inFlightRequestCount();
    }
}

 

# firePendingCompletedRequests()： 触发pendingCompleted里的请求

private void firePendingCompletedRequests() {
    boolean completedRequestsFired = false;
    for (;;) {
        // 获取RequestFutureCompletionHandler
        RequestFutureCompletionHandler completionHandler = pendingCompletion.poll();
        // 如果为空，跳出循环
        if (completionHandler == null)
            break;

        completionHandler.fireCompletion();
        completedRequestsFired = true;
    }

    // 为了防止NetworkClient在poll方法中被阻塞，唤醒这个NetworkClient
    if (completedRequestsFired)
        client.wakeup();
}

 

# checkDisconnects(long now)： 检测连接状态，检测消费者与每个Node之间的连接状态，当检测到连接断开的Node时，会将其在unsent列表中对应的全部Client

private void checkDisconnects(long now) {
    Iterator<Map.Entry<Node, List<ClientRequest>>> iterator = unsent.entrySet().iterator();
    while (iterator.hasNext()) {
        Map.Entry<Node, List<ClientRequest>> requestEntry = iterator.next();
        Node= requestEntry.getKey();
        if (client.connectionFailed(node)) {
            // Remove entry before invoking request callback to avoid callbacks handling
            // coordinator failures traversing the unsent list again.
            iterator.remove();
            for (ClientRequest request : requestEntry.getValue()) {
                RequestFutureCompletionHandler handler =
                        (RequestFutureCompletionHandler) request.callback();
                handler.onComplete(new ClientResponse(request, now, true, null));
            }
        }
    }
}

 

# failExpiredRequests(long now)： unsent中的超时请求，它会遍历整个unsent集合，检测，每一个ClientRequest是否超时，调用超时ClientRequest的回调函数，并将其从unsent列表删除

private void failExpiredRequests(long now) {
    // clear all expired unsent requests and fail their corresponding futures
    Iterator<Map.Entry<Node, List<ClientRequest>>> iterator = unsent.entrySet().iterator();
    while (iterator.hasNext()) {
        Map.Entry<Node, List<ClientRequest>> requestEntry = iterator.next();
        Iterator<ClientRequest> requestIterator = requestEntry.getValue().iterator();
        while (requestIterator.hasNext()) {
            ClientRequest request = requestIterator.next();
            if (request.createdTimeMs() < now - unsentExpiryMs) {
                RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) request.callback();
                handler.onFailure(new TimeoutException("Failed to send request after " + unsentExpiryMs + " ms."));
                requestIterator.remove();
            } else
                break;
        }
        if (requestEntry.getValue().isEmpty())
            iterator.remove();
    }
}

# trySend(long now)： 尝试发送请求

private boolean trySend(long now) {
    // send any requests that can be sent now
    boolean requestsSent = false;
    // 遍历unsent列表里的ClientRequest，判断NetworkClient是否已准备好，如果准备好了就开始发送请求
    for (Map.Entry<Node, List<ClientRequest>> requestEntry: unsent.entrySet()) {
        Node= requestEntry.getKey();
        Iterator<ClientRequest> iterator = requestEntry.getValue().iterator();
        while (iterator.hasNext()) {
            ClientRequest request = iterator.next();
            if (client.ready(node, now)) {
                client.send(request, now);
                iterator.remove();// 从unsent集合列表删除该请求
                requestsSent = true;
            }
        }
    }
    return requestsSent;
}

 

# maybeTriggerWakeup()： 检测wakeup 和 wakeupDisabledCount,查看是否有其他线程中断，如果有中断请求，则抛出WakeupExeception

private void maybeTriggerWakeup() {
    if (wakeupDisabledCount == 0 && wakeup.get()) {
        wakeup.set(false);
        throw new WakeupException();
    }
}

 

内部类

RequestFutureCompletionHandler：一个回调接口的实现，用于在请求完成时根据相应执行该回调方法。如果在处理请求时存在断开连接，那么这个处理程序也会被调用。

// 请求结果，可以判断请求是否完成，请求成功还是失败，请求是否可重试
private final RequestFuture<ClientResponse> future;
// 响应结果
private ClientResponse response;

// 当请求完成的时候，触发某些动作
public void fireCompletion() {
    if (e != null) {
        future.raise(e);
    } else if (response.wasDisconnected()) {
        ClientRequest request = response.request();
        RequestSend send = request.request();
        ApiKeys api = ApiKeys.forId(send.header().apiKey());
        int correlation = send.header().correlationId();
        log.debug("Cancelled {} request {} with correlation id {} due to node {} being disconnected",
                api, request, correlation, send.destination());
        future.raise(DisconnectException.INSTANCE);
    } else {
        future.complete(response);
    }
}