ConsumerNetworkClient 是对NetworkClient的一个封装,提供了额外的一些功能,首先我们看一下它有哪些属性:
# NetworkClient
private final KafkaClient client;
# 等待被发送的请求队列
private final Map<Node,List<ClientRequest>> unsent = new HashMap<>();
# 元数据信息
private final Metadata metadata;
# 重试时间
private final long retryBackoffMs;
# ClientRequest在unsent列表的超时时间
private final long unsentExpiryMs;
# KafkaConsumer是否正在执行不可中断的方法的次数,每次进入一个不可中断方法就加1,退出减1,它只会被KafkaConsumer线程修改,其他线程不能修改
private int wakeupDisabledCount = 0;
# 当请求完成时,它们会在调用之前被转移到这个队列中。这样做的目的是为了避免在持有该对象的监视器时调用它们,从而为死锁打开大门。
private finalConcurrentLinkedQueue<RequestFutureCompletionHandler> pendingCompletion =new ConcurrentLinkedQueue<>();
接着我们看一些它的核心方法:
# poll(long timeout, long now, PollConditionpollCondition) 主要用于发送请求,检测连接状态,处理超时请求
public void poll(long timeout, long now, PollCondition pollCondition) {
// 触发pendingCompleted里的请求
firePendingCompletedRequests();
synchronized (this) {
// 发送现在我们能发送的所有请求
trySend(now);
// 检测我们是否还需要poll
if (pollCondition == null || pollCondition.shouldBlock()) {
// if thereare no requests in flight, do not block longer than the retry backoff
if (client.inFlightRequestCount() == 0)
timeout = Math.min(timeout, retryBackoffMs);
client.poll(Math.min(MAX_POLL_TIMEOUT_MS, timeout), now);
now = time.milliseconds();
} else {
client.poll(0, now);
}
// 检测连接状态,检测消费者与每个Node之间的连接状态,当检测到连接断开的Node时,会将其在unsent列表中对应的全部Client
// Request对象清除掉,之后调用这些ClientRequest函数,并且设置disconnect标记为true
checkDisconnects(now);
// 检测wakeup 和wakeupDisabledCount,查看是否有其他线程中断,如果有中断请求,则抛出WakeupExeception
maybeTriggerWakeup();
// 再次试图发送,可能现在内存已经清理了或者Node可以连接上了
trySend(now);
// 处理unsent中的超时请求,它会遍历整个unsent集合,检测,每一个ClientRequest是否超时,调用超时ClientRequest
// 的回调函数,并将其从unsent列表删除
failExpiredRequests(now);
}
// 触发pendingCompleted里的请求
firePendingCompletedRequests();
}
# poll(RequestFuture<?> future, longtimeout): 是一个实现阻塞发送请求的功能
public boolean poll(RequestFuture<?> future, long timeout) { long begin = time.milliseconds(); long remaining = timeout; long now = begin; do { poll(remaining, now, future);// 请求未完成则调用poll方法 now = time.milliseconds(); long elapsed = now - begin; remaining = timeout - elapsed; } while (!future.isDone() && remaining > 0); //循环检测future的完成情况 return future.isDone(); }
# send(Node, ApiKeys api, shortversion,AbstractRequest request):
会将等待发送的请求封装成ClinetRequest
private RequestFuture<ClientResponse> send(Node node, ApiKeys api, short version, AbstractRequest request) { long now = time.milliseconds(); RequestFutureCompletionHandler completionHandler = new RequestFutureCompletionHandler(); RequestHeader header = client.nextRequestHeader(api, version); RequestSend send = new RequestSend(node.idString(), header, request.toStruct()); // 将这个ClinetRequest放到unsent列表 put(node, new ClientRequest(now, true, send, completionHandler)); // 如果NetworkCLient被poll阻塞,则唤醒它 client.wakeup(); // 返回RequestFuture对象 return completionHandler.future; }
# put(Node node, ClientRequest request): 往unsent列表添加ClientRequest 这些请求等待被发送
private void put(Node node, ClientRequest request) { synchronized (this) { List<ClientRequest> nodeUnsent = unsent.get(node); if (nodeUnsent == null) { nodeUnsent = new ArrayList<>(); unsent.put(node, nodeUnsent); } nodeUnsent.add(request); } }
# leastLoadedNode: 获取集群中负载最少的节点
public Node leastLoadedNode() { synchronized (this) { return client.leastLoadedNode(time.milliseconds()); } }
# awaitMetadataUpdate(): 除非元数据已经更新,否则一直阻塞
public void awaitMetadataUpdate() { awaitMetadataUpdate(Long.MAX_VALUE); }
# awaitMetadataUpdate(long timeout): 除非元数据已经更新,否则在指定的时间内一直阻塞
public boolean awaitMetadataUpdate(long timeout) { long startMs = time.milliseconds(); int version = this.metadata.requestUpdate(); do { poll(timeout); } while (this.metadata.version() == version && time.milliseconds() - startMs < timeout); return this.metadata.version() > version; }
# ensureFreshMetadata(): 确保元数据应经刷新
public void ensureFreshMetadata() { if (this.metadata.updateRequested() || this.metadata.timeToNextUpdate(time.milliseconds()) == 0) awaitMetadataUpdate(); }
# awaitPendingRequests(Node node): 等待指定节点的pedning的请求已经完成,否则一直阻塞
public void awaitPendingRequests(Node node) { while (pendingRequestCount(node) > 0) poll(retryBackoffMs); }
# pendingRequestCount(Node node): 获得指定节点pending的request数量
public int pendingRequestCount(Node node) { synchronized (this) { List<ClientRequest> pending = unsent.get(node); int unsentCount = pending == null ? 0 : pending.size(); return unsentCount + client.inFlightRequestCount(node.idString()); } }
# pendingRequestCount(): 获得所有节点pending的request数量
public int pendingRequestCount() { synchronized (this) { int total = 0; for (List<ClientRequest> requests: unsent.values()) total += requests.size(); return total + client.inFlightRequestCount(); } }
# firePendingCompletedRequests(): 触发pendingCompleted里的请求
private void firePendingCompletedRequests() { boolean completedRequestsFired = false; for (;;) { // 获取RequestFutureCompletionHandler RequestFutureCompletionHandler completionHandler = pendingCompletion.poll(); // 如果为空,跳出循环 if (completionHandler == null) break; completionHandler.fireCompletion(); completedRequestsFired = true; } // 为了防止NetworkClient在poll方法中被阻塞,唤醒这个NetworkClient if (completedRequestsFired) client.wakeup(); }
# checkDisconnects(long now): 检测连接状态,检测消费者与每个Node之间的连接状态,当检测到连接断开的Node时,会将其在unsent列表中对应的全部Client
private void checkDisconnects(long now) { Iterator<Map.Entry<Node, List<ClientRequest>>> iterator = unsent.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<Node, List<ClientRequest>> requestEntry = iterator.next(); Node= requestEntry.getKey(); if (client.connectionFailed(node)) { // Remove entry before invoking request callback to avoid callbacks handling // coordinator failures traversing the unsent list again. iterator.remove(); for (ClientRequest request : requestEntry.getValue()) { RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) request.callback(); handler.onComplete(new ClientResponse(request, now, true, null)); } } } }
# failExpiredRequests(long now): unsent中的超时请求,它会遍历整个unsent集合,检测,每一个ClientRequest是否超时,调用超时ClientRequest的回调函数,并将其从unsent列表删除
private void failExpiredRequests(long now) { // clear all expired unsent requests and fail their corresponding futures Iterator<Map.Entry<Node, List<ClientRequest>>> iterator = unsent.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<Node, List<ClientRequest>> requestEntry = iterator.next(); Iterator<ClientRequest> requestIterator = requestEntry.getValue().iterator(); while (requestIterator.hasNext()) { ClientRequest request = requestIterator.next(); if (request.createdTimeMs() < now - unsentExpiryMs) { RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) request.callback(); handler.onFailure(new TimeoutException("Failed to send request after " + unsentExpiryMs + " ms.")); requestIterator.remove(); } else break; } if (requestEntry.getValue().isEmpty()) iterator.remove(); } }
# trySend(long now): 尝试发送请求
private boolean trySend(long now) { // send any requests that can be sent now boolean requestsSent = false; // 遍历unsent列表里的ClientRequest,判断NetworkClient是否已准备好,如果准备好了就开始发送请求 for (Map.Entry<Node, List<ClientRequest>> requestEntry: unsent.entrySet()) { Node= requestEntry.getKey(); Iterator<ClientRequest> iterator = requestEntry.getValue().iterator(); while (iterator.hasNext()) { ClientRequest request = iterator.next(); if (client.ready(node, now)) { client.send(request, now); iterator.remove();// 从unsent集合列表删除该请求 requestsSent = true; } } } return requestsSent; }
# maybeTriggerWakeup(): 检测wakeup 和 wakeupDisabledCount,查看是否有其他线程中断,如果有中断请求,则抛出WakeupExeception
private void maybeTriggerWakeup() { if (wakeupDisabledCount == 0 && wakeup.get()) { wakeup.set(false); throw new WakeupException(); } }
内部类
RequestFutureCompletionHandler:一个回调接口的实现,用于在请求完成时根据相应执行该回调方法。如果在处理请求时存在断开连接,那么这个处理程序也会被调用。
// 请求结果,可以判断请求是否完成,请求成功还是失败,请求是否可重试 private final RequestFuture<ClientResponse> future; // 响应结果 private ClientResponse response;
// 当请求完成的时候,触发某些动作 public void fireCompletion() { if (e != null) { future.raise(e); } else if (response.wasDisconnected()) { ClientRequest request = response.request(); RequestSend send = request.request(); ApiKeys api = ApiKeys.forId(send.header().apiKey()); int correlation = send.header().correlationId(); log.debug("Cancelled {} request {} with correlation id {} due to node {} being disconnected", api, request, correlation, send.destination()); future.raise(DisconnectException.INSTANCE); } else { future.complete(response); } }