RaftLog输出流
1. BufferedChannel
在FileChannel之上提供了缓冲层,应该是为了避免频繁的I/O操作,解决何时触发FileChannel.force(boolean)方法
1. 实例参数
class BufferedWriteChannel implements Closeable {
//1. Channel
private final FileChannel fileChannel;
//2. ByteBuffer,用来缓存写入channel的数据
private final ByteBuffer writeBuffer;
//3. 用来表示是否调用过FileChannel.force()方法,即数据是否已同步到磁盘上
private boolean forced = true;
//4.
private final Supplier<CompletableFuture<Void>> flushFuture;
}
2. 构造器
就是简单的赋值构造器
3. 实例方法
这里核心的是两类方法:
- flush()/asyncFlush()
- flushBuffer()
flushBuffer()方法会将writeBuffer中的数据通过fileChannel.write(ByteBuffer)方法写入到FileChannel中,但是这并没有保证数据落盘,数据落盘需要通过调用flush()/asyncFlush(ExecutorService)方法。需要注意的是,flushBuffer()方法会将forced实例变量改为false,用来提示flush()/asyncFlush(ExecutorService)方法触发fileChanne.sync(Boolean)方法,完成数据落盘。这里的forced也可以理解为避免在没有数据写入时重复调用FileChannel.force(boolean)方法
class BufferedWriteChannel implements Closeable {
private void flushBuffer() throws IOException {
if(writeBuffer.position() == 0) {
return; //noting to flush
}
writeBuffer.flip();
do {
fileChannel.write(writeBuffer);
} while(writeBuffer.hasRemaining());
writeBuffer.clear();
forced = false;
}
/**
Write any data in the buffer to the file and force a sync operation so that data is persisted to the disk.
**/
void flush() throws IOException{
flushBuffer();
if(!forced) {
fileChannel.force(false);
forced = true;
}
}
private void asyncFlush(ExecutorService executor) {
flushBuffer();
if(forced) {
return CompletableFuture.completedFuture(null);
}
final CompletableFuture<Void> f = CompletableFuture.supplyAsync(this::fileChannelForce, executor);
forced = true;
return f;
}
private Void fileChannelForce() {
try {
fileChannel.force(false);
} catch(IOException) {
throw new CompletionException(e);
}
retrun null;
}
}
这里asyncFlush(ExecutorService)方法之所以调用fileChannelForce(),是因为Supplier接口不支持方法Throwable。所以要使用try…catch语句进行包裹
接下来关注其他实例方法
class BufferedWriteChannel implements Closeable {
void write(byte[] b) throws IOException {
int offset = 0;
while(offset < b.length){
int toPut = Math.min(b.length - offset, writeBuffer.remaining());
writeBuffer.put(b, offset, toPut);
offset += toPut;
if(writeBuffer.remaining() == 0){
flushBuffer();
}
}
}
void preallocateIfNecessary(long size, CheckedBiFunction<FileChannel, long, long, IOException> preallocate) throws IOException {
final long outstanding = writeBuffer.position() + size;
if(fileChannel.position() + outstanding > fileChannel.size()) {
preallocate.appy(fileChannel, outstanding);
}
}
boolean isOpen() {
return fileChannel.isOpen();
}
@Override
public void close() throws IOEception {
if(!isOpen()) {
return;
}
try {
Optional.ofNullable(flushFuture)
.ifPresent(f -> f.get());
fileChannel.truncate(fileChannel.position());
} finally {
fileChannel.close();
}
}
}
4.静态方法
class BufferedWriteChannel implements Closeable {
static BufferedWriteChannel open(File file, boolean append, ByteBuffer buffer, Supplier<CompletableFuture<Void> flushFuture) throws IOException {
final RandomAccessFile raf = new RandomAccessFile(file, "rw");
final FileChannel fc = raf.getChannel();
if(append) {
fc.position(fc.size());
} else {
fc.truncate(0);
}
Preconditions.assertSame(fc.size(), fc.position(), "fc.position");
return new BufferedWriteChannel(fc, buffer, flushFuture);
}
}
2. SegmentedRaftLogOutputStream
1. 实例变量
public class SegmentedRaftLogOutputStream implements Closeable {
private final File file;
private final BufferedWriteChannel out;; //buffered FileChannel for writing.
private final Checksum checksum;
private final long segmentMaxSize;
private final long prealloacatdSize;
}
2. 静态变量
public class SegmentedRaftLogOutputStream implements Closeable {
private static final ByteBuffer FILL;
private static final int BUFFER_SIZE = 1024 * 1024; //1MB
static {
FILE = BytBuffer.allocate(BUFFER_SIZE);
for(int i = 0; i < FILL.capacity(); i ++) {
FILL.put(SegmentedRaftLogFormat.getTerminator()); //0
}
FILL.flip(); //limit -> position && position -> 0
}
}
3. 构造器
public class SegmentedRaftLogOutputStream implements Closeable {
public SegmentedRaftLogOutputStream(File file, boolean append, long segmentMaxSize, long preallocateSize, ByteBuffer byteBuffer, Supplier<CompletableFuture<Void> flushFuture) throws IOException {
this.file = file;
this.checkSum = new PureJavaCrc32C();
this.segmentMaxSize = segmentMaxSize;
this.preallocatedSize = preallocatedSize;
this.out = BufferedWriteChannel.open(file, append, byteBuffer, flushFuture);
//如果不是以append方式打开文件,那么先将Header信息写入到文件中
if(!append) {
preallocateIfNecessary(SegmentedRaftLogFormat.getHeaderLength());
SegmentedRaftLogFormat.applyHeaderTo(CheckedConsumber.asCheckedFunction(out::write));
out.flush();
}
}
}
4. 实例方法
这里核心的是write(LogEntryProto)方法,该方法决定了RaftLog文件的格式
Format:
- The serialized size of the entry.
- The entry
- 4-byte checksum of the entry.
Size in bytes to be writen:
(size to encode n) + n + (checksum size), where n is the entry serialized size and the checksum size is 4.
这里需要说明的是,n是entry的size大小,然后使用Protobuf变长编码对n进行序列化。在Java中,整型使用4字节,但是在Protobuf变长编码中,整型使用1-5字节,越小的数字使用的字节数越少。这里如此设计也能稍稍节省空间
public class SegmentedRaftLogOutputStream implements Closeable {
public void write(LogEntryProto entry) throws IOException {
final int serialized = entry.getSerializedSize();
//第一个操作数便是使用Protobuf对entrySize进行编码后的字节数
final int proto = CodedOutputStream.computUInt32SizeNoTag(serialized) + serialized;
final byte[] buf = new byte[proto + 4]; //4-byte checksum
preallocateIfNecessary(buf.length);
CodedOutputStream cout = CodedOutputStream.newInstance(buf);
//1. 将entrySize进行Protobuf编码后写入buffer
cout.writeUInt32NoTag(seriazlied);
//2. 将entry写入buffer
entry.writeTo(cout);
//3.将checksum写入buffer
checksum.reset();
checksum.update(buf, 0, proto);
ByteBuffer.wrap(buf, proto, 4).putInt((int) checksum.getValue());
//落地文件
out.write(buf);
}
@Override
public void close() throws IOException {
try {
flush();
} finally {
IOUtils.cleanup(LOG, out);
}
}
/**
Flush data to Persistent store.
Collect sync metrics.
**/
public void flush() throws IOException {
try {
out.flush();
} catch(IOException ioe) {
String msg = "Failed to flush " + this;
throw new IOException(msg, ioe);
}
}
CompletableFuture<Void> asyncFlush(ExecutorService executor) throws IOException {
try {
return out.asyncFlush(executor);
} catch (IOException ioe) {
String msg = "Failed to asyncFlush " + this;
throw new IOException(msg, ioe);
}
}
private long preallocate(FileChannel fc, long outstanding) throws IOException {
final long actual = actualPreallocateSize(outstanding, segentMaxSize - fc.size(), preallocatedSize);
Preconditions.assertTrue(actual >= outstanding);
final long allcoated = IOUtils.preallocate(fc, actual, FILL);
return allocated;
}
}
通过write方法可以看出RaftLog的文件组织形式为
5. 静态方法
public class SegmentedRaftLogOutputStream implements Closeable {
/**
如果待写数据size > 剩余空间,直接返回待写数据size
如果待写数据size < 剩余空间
1. 如果待写数据size > 预分配空间,返回待写数据size
2. 返回剩余空间和预分配空间的最小值
**/
private static long actualPreallocateSize(long outstandingData, long remainingSpace, long preallocate) {
return outstandingData > remainingSpace? outstandingData
: outstandingData > preallocate? outstandingData
: Math.min(preallocate, remaininigSpace);
}
}