RecordBatch分析

RecordBatch主要用于封装MemoryRecords以及其他的一些统计类型的信息。

一 核心字段

int recordCount: 保存的record个数

int maxRecordSize: 最大record的字节数

int attempts: 尝试发送当前RecordBatch的次数

long lastAttemptMs: 最后一次尝试发送的时间戳

long lastAppendTime: 最后一次向RecordBatch追加消息的时间

long createdMs: 创建时间

long drainedMs: 好费时间

MemoryRecords records: 用来存储records的对象

TopicPartition: 分区信息

ProduceRequestResult produceFuture: 标记RecordBatch状态的Future对象

List<Thunk> thunks: Thunk对象集合

long offsetCounter: 记录消息在RecordBatch中的偏移量

boolean retry: 是否正在重试，如果RecordBatch发送失败就会尝试重发

二 ProduceRequestResult和Thunk

ProduceRequestResult: 完成生产者请求的一个结果类，但是该类并没有根据并发库下的Future来实现而是根据CountDownLatch来实现。当RecordBatch中全部消息被正常响应，或超市或关闭生产者时，会调用done方法标记完成，可以通过error字段区分是异常完成还是正常完成

public final class ProduceRequestResult {

    private final CountDownLatch latch = new CountDownLatch(1);
    private volatile TopicPartition;
    private volatile long baseOffset = -1L;
    private volatile RuntimeException error;

    public ProduceRequestResult() {
    }

    public void done(TopicPartition, long baseOffset, RuntimeException error) {
        this.topicPartition = topicPartition;
        this.baseOffset = baseOffset;
        this.error = error;
        this.latch.countDown();
    }
    
    // 等待请求的完成
    public void await() throws InterruptedException {
        latch.await();
    }

    // 在指定超时时间范围内，等待请求的完成
    public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
        return latch.await(timeout, unit);
    }

    // 返回服务器端为RecordBatch中第一个消息分配的offset
    public long baseOffset() {
        return baseOffset;
    }


    // 处理请求是发生异常
    public RuntimeException error() {
        return error;
    }


    // 返回分区情况
    public TopicPartition topicPartition() {
        return topicPartition;
    }


    // 返回是否完成
    public boolean completed() {
        return this.latch.getCount() == 0L;
    }
}

 

Thunk: 主要是消息的回调函数和FutureRecordMetadata一个封装

final private static class Thunk {
    // 指向消息的回调函数
    final Callback callback;
    // 发送的record的future结果
    final FutureRecordMetadata future;

    public Thunk(Callback callback, FutureRecordMetadata future) {
        this.callback = callback;
        this.future = future;
    }
}

 

三 重要方法

3.1 tryAppend: 添加record到当前结果集，并返回FutureRecord元数据对象

public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) {
    // 检测MemoryRecords是否还可以容纳新的record，如果不行返回空
    if (!this.records.hasRoomFor(key, value)) {
        return null;
    } else {
        // 添加record 到buffer，并返回校验和
        long checksum = this.records.append(offsetCounter++, timestamp, key, value);
        // 和当前RecordBatch里的组最大记录比较，如果比当前的大就更新
        this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value));
        // 将最后添加时间更新为now
        this.lastAppendTime = now;
        FutureRecordMetadatafuture = new FutureRecordMetadata(this.produceFuture, this.recordCount,
                                                              timestamp, checksum,
                                                              key == null ? -1 : key.length,
                                                              value == null ? -1 : value.length);
        // 通过回调构造chunk对象并添加到thunks
        if (callback != null)
            thunks.add(new Thunk(callback, future));
        // 更新RecordBatch的记录数
        this.recordCount++;
        return future;
    }
}

3.2 done 当RecordBatch成功收到正常响应，或超时或关闭生产时，都会调用RecordBatch的done方法，该方法会回调RecordBatch中全部消息的callback回调函数，并且调用ProducerRequestResult的done方法

public void done(long baseOffset, long timestamp, RuntimeException exception) {
    log.trace("Produced messages to topic-partition {} with base offset offset {} and error: {}.",
              topicPartition, baseOffset, exception);
    // 执行RecordBatch中所有消息的回调
    for (int i = 0; i < this.thunks.size(); i++) {
        try {
            // 得到每一个消息的回调对应的Thunk对象
            Thunk thunk = this.thunks.get(i);
            if (exception == null) {
                // 如果没有异常，正常处理完成，将服务端返回的offset，timestamp和其他信息封装成RecordMetadata对象
                RecordMetadata metadata = new RecordMetadata(this.topicPartition,  baseOffset, thunk.future.relativeOffset(),
                    timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, thunk.future.checksum(),
                    thunk.future.serializedKeySize(), thunk.future.serializedValueSize());
                // 调用消息自定义的callback函数
                thunk.callback.onCompletion(metadata, null);
            } else {
                // 调用消息自定义的callback函数
                thunk.callback.onCompletion(null, exception);
            }
        } catch (Exception e) {
            log.error("Error executing user-provided callback on message for topic-partition {}:", topicPartition, e);
        }
    }
    // 标记这个生产者请求已经完成，countDownLatch减去1
    this.produceFuture.done(topicPartition, baseOffset, exception);
}

 

3.3 maybeExpire 检测RecordBatch是否到期

public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) {
    boolean expire = false;
    String errorMessage = null;
    // 检测是否到期
    if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) {
        expire = true;
        errorMessage = (now - this.lastAppendTime) + " ms has passed since last append";
    }
    else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) {
        expire = true;
        errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time";
    } else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) {
        expire = true;
        errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time";
    }
    // 如果到期，关闭MemoryRecord不再添加
    if (expire) {
        this.records.close();
        this.done(-1L, Record.NO_TIMESTAMP,
                  new TimeoutException("Expiring " + recordCount + " record(s) for " + topicPartition + " due to " + errorMessage));
    }

    return expire;
}