sender分析之创建请求

一 Sender run方法调用流程

# 从Metadata获取集群元数据

# 调用RecordAccumulator.ready方法，根据RecordAccumulator的缓存情况，选出可以向哪些Node发送消息，返回ReadyCheckResult对象

# 如果ReadyCheckResult存在某些分区没有leader副本，则调用Metadata的requestUpdate方法，标记需要更新kafka的集群信息

# 针对ReadyCheckResult的readyNodes集合，循环调用NetworkClient的ready方法，目的是检测网络I/O方面是否符合发送条件，不符合发送条件的Node将会从readyNodes集合中删除

# 调用RecordAccumulator的drain方法获取待发送的消息集合

# 调用RecordAccumulator的abortExpiredBatchers方法处理RecordAccumulator中超时的消息

# 调用Sender的createProduceRequests方法，将发送的消息封装成ClientRequest请求

# 调用NetworkClient.send方法，将ClientRequest写入KafkaChannel的send字段

# 调用NetworkClient的poll方法，将KafkaChannel中send字段保存的ClientRequest发送出去，同时还会处理服务端发回的响应处理超时请求，调用用户自定义的函数等

void run(long now) {
    // 从Metadata获取集群元数据
    Cluster cluster = metadata.fetch();
    // 根据RecordAccumulator的缓存情况，选出可以向哪些Node发送消息，返回ReadyCheckResult对象
    RecordAccumulator.ReadyCheckResultresult = this.accumulator.ready(cluster, now);
    // 如果ReadyCheckResult存在某些分区没有leader副本，则调用Metadata的requestUpdate方法，标记需要更新kafka的集群信息
    if (!result.unknownLeaderTopics.isEmpty()) {
        for (String topic : result.unknownLeaderTopics)
            this.metadata.add(topic);
        this.metadata.requestUpdate();
    }

    // 针对ReadyCheckResult的readyNodes集合，循环调用NetworkClient的ready方法，
    // 目的是检测网络I/O方面是否符合发送条件，不符合发送条件的Node将会从readyNodes集合中删除
    Iterator<Node> iter = result.readyNodes.iterator();
    long notReadyTimeout = Long.MAX_VALUE;
    while (iter.hasNext()) {
        Node node = iter.next();
        if (!this.client.ready(node, now)) {
            iter.remove();
            notReadyTimeout= Math.min(notReadyTimeout, this.client.connectionDelay(node, now));
        }
    }

    // 调用RecordAccumulator的drain方法获取待发送的消息集合
    Map<Integer, List<RecordBatch>> batches = this.accumulator.drain(cluster, result.readyNodes,
        this.maxRequestSize, now);
    // 是否需要保证消息的顺序
    if (guaranteeMessageOrder) {
        // 遍历record batch
        for (List<RecordBatch> batchList : batches.values()) {
            for (RecordBatch batch : batchList)
                this.accumulator.mutePartition(batch.topicPartition);
        }
    }
    // 调用RecordAccumulator的abortExpiredBatchers方法处理RecordAccumulator中超时的消息
    List<RecordBatch> expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout, now);

    for (RecordBatch expiredBatch : expiredBatches)
        this.sensors.recordErrors(expiredBatch.topicPartition.topic(), expiredBatch.recordCount);

    sensors.updateProduceRequestMetrics(batches);
    // 创建生产者请求
    List<ClientRequest> requests = createProduceRequests(batches, now);

    long pollTimeout = Math.min(result.nextReadyCheckDelayMs, notReadyTimeout);
    if (result.readyNodes.size() > 0) {
        log.trace("Nodes with data ready tosend: {}", result.readyNodes);
        log.trace("Created {} producerequests: {}", requests.size(), requests);
        pollTimeout = 0;
    }
    // 将ClientRequest写入KafkaChannel的send字段
    for (ClientRequest request : requests)
        client.send(request, now);
    // 调用NetworkClient的poll方法，将KafkaChannel中send字段保存的ClientRequest发送出去，
    // 同时还会处理服务端发回的响应处理超时请求，调用用户自定义的函数等
    this.client.poll(pollTimeout, now);
}

 

二 创建请求

我们先分析ProduceRequest和ProduceResponse消息体格式：

api_key: API标识

api_version: API版本号

correaltion_id: 一个单调递增序号

client_id: 客户端id

acks: 确认机制，0 不需要确认，1 只需要leader确认，-1所有副本都需要确认

timeout: 超时时间

topic: topic名称

partition: partition编号

record_set: 消息

correaltion_id: 一个单调递增序号

topic: topic名称

partition: partition编号

error_code: 错误码

base_offset: 服务端为消息生成的一个offset

timestamp: 瞬间戳

throttle_time_ms: 延长时间

 

private List<ClientRequest> createProduceRequests(Map<Integer, List<RecordBatch>> collated, long now) {
    // 保存创建的ClientRequest列表
    List<ClientRequest> requests = new ArrayList<ClientRequest>(collated.size());
    for (Map.Entry<Integer, List<RecordBatch>> entry : collated.entrySet())
        // 将发往同一个Node的RecordBatch封装成ClientRequest
        requests.add(produceRequest(now, entry.getKey(), acks, requestTimeout, entry.getValue()));
    return requests;
}

 

private ClientRequest produceRequest(long now, int destination, short acks, int timeout, List<RecordBatch> batches) {
    Map<TopicPartition, ByteBuffer> produceRecordsByPartition = new HashMap<TopicPartition, ByteBuffer>(batches.size());
    final Map<TopicPartition, RecordBatch> recordsByPartition = new HashMap<TopicPartition, RecordBatch>(batches.size());
    // 将RecordBatch按照partiiton分类，同时构建集合
    for (RecordBatch batch : batches) {
        TopicPartition tp = batch.topicPartition;
        produceRecordsByPartition.put(tp, batch.records.buffer());
        recordsByPartition.put(tp, batch);
    }
    // 创建ProduceRequest和RequestSend
    ProduceRequest request = new ProduceRequest(acks, timeout, produceRecordsByPartition);
    RequestSend send = new RequestSend(Integer.toString(destination),
                                       this.client.nextRequestHeader(ApiKeys.PRODUCE),
                                       request.toStruct());
    // 创建RequestCompletionHandler作为回调对象
    RequestCompletionHandler callback = new RequestCompletionHandler() {
        public void onComplete(ClientResponse response) {
            handleProduceResponse(response, recordsByPartition, time.milliseconds());
        }
    };
    // 创建ClientRequest对象
    return new ClientRequest(now, acks != 0, send, callback);
}