RecordBatch主要用于封装MemoryRecords以及其他的一些统计类型的信息。
一 核心字段
int recordCount: 保存的record个数
int maxRecordSize: 最大record的字节数
int attempts: 尝试发送当前RecordBatch的次数
long lastAttemptMs: 最后一次尝试发送的时间戳
long lastAppendTime: 最后一次向RecordBatch追加消息的时间
long createdMs: 创建时间
long drainedMs: 好费时间
MemoryRecords records: 用来存储records的对象
TopicPartition: 分区信息
ProduceRequestResult produceFuture: 标记RecordBatch状态的Future对象
List<Thunk> thunks: Thunk对象集合
long offsetCounter: 记录消息在RecordBatch中的偏移量
boolean retry: 是否正在重试,如果RecordBatch发送失败就会尝试重发
二 ProduceRequestResult和Thunk
ProduceRequestResult: 完成生产者请求的一个结果类,但是该类并没有根据并发库下的Future来实现而是根据CountDownLatch来实现。当RecordBatch中全部消息被正常响应,或超市或关闭生产者时,会调用done方法标记完成,可以通过error字段区分是异常完成还是正常完成
public final class ProduceRequestResult {
private final CountDownLatch latch = new CountDownLatch(1);
private volatile TopicPartition;
private volatile long baseOffset = -1L;
private volatile RuntimeException error;
public ProduceRequestResult() {
}
public void done(TopicPartition, long baseOffset, RuntimeException error) {
this.topicPartition = topicPartition;
this.baseOffset = baseOffset;
this.error = error;
this.latch.countDown();
}
// 等待请求的完成
public void await() throws InterruptedException {
latch.await();
}
// 在指定超时时间范围内,等待请求的完成
public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
return latch.await(timeout, unit);
}
// 返回服务器端为RecordBatch中第一个消息分配的offset
public long baseOffset() {
return baseOffset;
}
// 处理请求是发生异常
public RuntimeException error() {
return error;
}
// 返回分区情况
public TopicPartition topicPartition() {
return topicPartition;
}
// 返回是否完成
public boolean completed() {
return this.latch.getCount() == 0L;
}
}
Thunk: 主要是消息的回调函数和FutureRecordMetadata一个封装
final private static class Thunk {
// 指向消息的回调函数
final Callback callback;
// 发送的record的future结果
final FutureRecordMetadata future;
public Thunk(Callback callback, FutureRecordMetadata future) {
this.callback = callback;
this.future = future;
}
}
三 重要方法
3.1 tryAppend: 添加record到当前结果集,并返回FutureRecord元数据对象
public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Callback callback, long now) {
// 检测MemoryRecords是否还可以容纳新的record,如果不行返回空
if (!this.records.hasRoomFor(key, value)) {
return null;
} else {
// 添加record 到buffer,并返回校验和
long checksum = this.records.append(offsetCounter++, timestamp, key, value);
// 和当前RecordBatch里的组最大记录比较,如果比当前的大就更新
this.maxRecordSize = Math.max(this.maxRecordSize, Record.recordSize(key, value));
// 将最后添加时间更新为now
this.lastAppendTime = now;
FutureRecordMetadatafuture = new FutureRecordMetadata(this.produceFuture, this.recordCount,
timestamp, checksum,
key == null ? -1 : key.length,
value == null ? -1 : value.length);
// 通过回调构造chunk对象并添加到thunks
if (callback != null)
thunks.add(new Thunk(callback, future));
// 更新RecordBatch的记录数
this.recordCount++;
return future;
}
}
3.2 done 当RecordBatch成功收到正常响应,或超时或关闭生产时,都会调用RecordBatch的done方法,该方法会回调RecordBatch中全部消息的callback回调函数,并且调用ProducerRequestResult的done方法
public void done(long baseOffset, long timestamp, RuntimeException exception) {
log.trace("Produced messages to topic-partition {} with base offset offset {} and error: {}.",
topicPartition, baseOffset, exception);
// 执行RecordBatch中所有消息的回调
for (int i = 0; i < this.thunks.size(); i++) {
try {
// 得到每一个消息的回调对应的Thunk对象
Thunk thunk = this.thunks.get(i);
if (exception == null) {
// 如果没有异常,正常处理完成,将服务端返回的offset,timestamp和其他信息封装成RecordMetadata对象
RecordMetadata metadata = new RecordMetadata(this.topicPartition, baseOffset, thunk.future.relativeOffset(),
timestamp == Record.NO_TIMESTAMP ? thunk.future.timestamp() : timestamp, thunk.future.checksum(),
thunk.future.serializedKeySize(), thunk.future.serializedValueSize());
// 调用消息自定义的callback函数
thunk.callback.onCompletion(metadata, null);
} else {
// 调用消息自定义的callback函数
thunk.callback.onCompletion(null, exception);
}
} catch (Exception e) {
log.error("Error executing user-provided callback on message for topic-partition {}:", topicPartition, e);
}
}
// 标记这个生产者请求已经完成,countDownLatch减去1
this.produceFuture.done(topicPartition, baseOffset, exception);
}
3.3 maybeExpire 检测RecordBatch是否到期
public boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) {
boolean expire = false;
String errorMessage = null;
// 检测是否到期
if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime)) {
expire = true;
errorMessage = (now - this.lastAppendTime) + " ms has passed since last append";
}
else if (!this.inRetry() && requestTimeoutMs < (now - (this.createdMs + lingerMs))) {
expire = true;
errorMessage = (now - (this.createdMs + lingerMs)) + " ms has passed since batch creation plus linger time";
} else if (this.inRetry() && requestTimeoutMs < (now - (this.lastAttemptMs + retryBackoffMs))) {
expire = true;
errorMessage = (now - (this.lastAttemptMs + retryBackoffMs)) + " ms has passed since last attempt plus backoff time";
}
// 如果到期,关闭MemoryRecord不再添加
if (expire) {
this.records.close();
this.done(-1L, Record.NO_TIMESTAMP,
new TimeoutException("Expiring " + recordCount + " record(s) for " + topicPartition + " due to " + errorMessage));
}
return expire;
}