kafka消费者如何读同一生产者消息_kafka 消费者拉取消息

本文只跟踪消费者拉取消息的流程。对于 java 客户端， kafka 的生产者和消费者复用同一个网络 io 类 NetworkClient。

入口在 KafkaConsumer#pollOnce 中，抽出主要步骤：

//构造 FetchRequest 请求，将请求对象放入 unsent 集合，等待发送

fetcher.sendFetches();//取出 unsent 中的请求，调用 NetworkClient#send，NetworkClinet#poll

client.poll(pollTimeout, nowMs, newPollCondition() {

@Overridepublic booleanshouldBlock() {//since a fetch might be completed by the background thread, we need this poll condition//to ensure that we do not block unnecessarily in poll()

return !fetcher.hasCompletedFetches();

}

});//返回数据给用户

return fetcher.fetchedRecords();

Fetcher#sendFetches

public synchronized intsendFetches() {//构造拉取消息请求。从哪个节点，哪个分区，什么位置拉取消息

Map fetchRequestMap =prepareFetchRequests();for (Map.Entryentry : fetchRequestMap.entrySet()) {final Node fetchTarget =entry.getKey();final FetchSessionHandler.FetchRequestData data =entry.getValue();//1. 借助 Builder 构造 FetchRequest 对象

final FetchRequest.Builder request =FetchRequest.Builder

.forConsumer(this.maxWaitMs, this.minBytes, data.toSend())

.isolationLevel(isolationLevel)

.setMaxBytes(this.maxBytes)

.metadata(data.metadata())

.toForget(data.toForget());if(log.isDebugEnabled()) {

log.debug("Sending {} {} to broker {}", isolationLevel, data.toString(), fetchTarget);

}

client.send(fetchTarget, request)//4. 给 RequestFutureCompletionHandler.future 添加 RequestFutureListener

.addListener(new RequestFutureListener() {

@Overridepublic voidonSuccess(ClientResponse resp) {synchronized (Fetcher.this) {

FetchResponse response=(FetchResponse) resp.responseBody();

FetchSessionHandler handler=sessionHandler(fetchTarget.id());if (handler == null) {

log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.",

fetchTarget.id());return;

}if (!handler.handleResponse(response)) {return;

}

Set partitions = new HashSet<>(response.responseData().keySet());

FetchResponseMetricAggregator metricAggregator= newFetchResponseMetricAggregator(sensors, partitions);for (Map.Entryentry : response.responseData().entrySet()) {

TopicPartition partition=entry.getKey();long fetchOffset =data.sessionPartitions().get(partition).fetchOffset;

FetchResponse.PartitionData fetchData=entry.getValue();

log.debug("Fetch {} at offset {} for partition {} returned fetch data {}",

isolationLevel, fetchOffset, partition, fetchData);//10. 把数据放入 completedFetches，最终返回给用户

completedFetches.add(newCompletedFetch(partition, fetchOffset, fetchData, metricAggregator,

resp.requestHeader().apiVersion()));

}

sensors.fetchLatency.record(resp.requestLatencyMs());

}

}

@Overridepublic voidonFailure(RuntimeException e) {synchronized (Fetcher.this) {

FetchSessionHandler handler=sessionHandler(fetchTarget.id());if (handler != null) {

handler.handleError(e);

}

}

}

});

}returnfetchRequestMap.size();

}

ConsumerNetworkClient#send

public RequestFuture send(Node node, AbstractRequest.Builder>requestBuilder) {long now =time.milliseconds();//2. 使用 RequestFutureCompletionHandler 作为回调函数

RequestFutureCompletionHandler completionHandler = newRequestFutureCompletionHandler();

ClientRequest clientRequest= client.newClientRequest(node.idString(), requestBuilder, now, true,

completionHandler);//3. 请求放入 unsent 集合

unsent.put(node, clientRequest);//wakeup the client in case it is blocking in poll so that we can send the queued request

client.wakeup();returncompletionHandler.future;

}

ConsumerNetworkClient#poll

//5. 发送 unsent 中的请求，并没有产生网络 io

trySend(now);//真实的网络数据写和读//6. 发送请求//7. 接收响应//8. 触发 RequestFutureCompletionHandler 回调

client.poll(0, now);//9. 触发 RequestFutureListener 中的回调

firePendingCompletedRequests();

NetworkClient#handleCompletedReceives

private void handleCompletedReceives(List responses, longnow) {for (NetworkReceive receive : this.selector.completedReceives()) {

String source=receive.source();

InFlightRequest req=inFlightRequests.completeNext(source);

Struct responseStruct=parseStructMaybeUpdateThrottleTimeMetrics(receive.payload(), req.header,

throttleTimeSensor, now);if(log.isTraceEnabled()) {

log.trace("Completed receive from node {} for {} with correlation id {}, received {}", req.destination,

req.header.apiKey(), req.header.correlationId(), responseStruct);

}

AbstractResponse body=AbstractResponse.parseResponse(req.header.apiKey(), responseStruct);if (req.isInternalRequest && body instanceofMetadataResponse)

metadataUpdater.handleCompletedMetadataResponse(req.header, now, (MetadataResponse) body);else if (req.isInternalRequest && body instanceofApiVersionsResponse)

handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) body);else

//此处给 responses 添加元素//return new ClientResponse(header, callback, destination, createdTimeMs, timeMs, false, null, response);//直接把请求的 callback 赋值给响应//生产者发送消息的 callback，是用户通过参数传入的//消费者拉取消息的 callback，是在 ConsumerNetworkClient#send 指定的，是 RequestFutureCompletionHandler

responses.add(req.completed(body, now));

}

}

NetworkClient#completeResponses

private void completeResponses(Listresponses) {for(ClientResponse response : responses) {try{//callback.onComplete(this);

response.onComplete();

}catch(Exception e) {

log.error("Uncaught error in request completion:", e);

}

}

}

RequestFutureCompletionHandler#onComplete

public voidonComplete(ClientResponse response) {this.response =response;

pendingCompletion.add(this);

}

ConsumerNetworkClient#firePendingCompletedRequests

private voidfirePendingCompletedRequests() {boolean completedRequestsFired = false;for(;;) {

RequestFutureCompletionHandler completionHandler=pendingCompletion.poll();if (completionHandler == null)break;

completionHandler.fireCompletion();

completedRequestsFired= true;

}//wakeup the client in case it is blocking in poll for this future‘s completion

if(completedRequestsFired)

client.wakeup();

}

ConsumerNetworkClient.RequestFutureCompletionHandler#fireCompletion

public voidfireCompletion() {if (e != null) {

future.raise(e);

}else if(response.wasDisconnected()) {

RequestHeader requestHeader=response.requestHeader();int correlation =requestHeader.correlationId();

log.debug("Cancelled {} request {} with correlation id {} due to node {} being disconnected",

requestHeader.apiKey(), requestHeader, correlation, response.destination());

future.raise(DisconnectException.INSTANCE);

}else if (response.versionMismatch() != null) {

future.raise(response.versionMismatch());

}else{

future.complete(response);

}

}

RequestFuture#complete

public voidcomplete(T value) {try{if (value instanceofRuntimeException)throw new IllegalArgumentException("The argument to complete can not be an instance of RuntimeException");if (!result.compareAndSet(INCOMPLETE_SENTINEL, value))throw new IllegalStateException("Invalid attempt to complete a request future which is already complete");

fireSuccess();

}finally{

completedLatch.countDown();

}

}private voidfireSuccess() {

T value=value();while (true) {

RequestFutureListener listener =listeners.poll();if (listener == null)break;//终于调到 RequestFutureListener

listener.onSuccess(value);

}

}

如果不考虑心跳线程，consumer 第一次 poll 是不会有数据的，此时请求才发出去，响应还没回来，必须在第二次 poll 时，才能同时处理网络读写事件。

跟完之后，个人觉得调用链还是挺长的。一点感觉，全程只有一个线程，但是每次走的分支都不一样，给人的启发就是，单线程只要不等待，速度也很快。

原文：https://www.cnblogs.com/allenwas3/p/11617514.html