leveldb之block

BlockBuilder

block的数据是通过BlockBuilder实现

定义

restarts_表示重启点，主要用来保护数据损坏。

class BlockBuilder {
public:
    explicit BlockBuilder(const Options* options);

    BlockBuilder(const BlockBuilder&) = delete;
    BlockBuilder& operator=(const BlockBuilder&) = delete;

    // Reset the contents as if the BlockBuilder was just constructed.
    void Reset();

    // REQUIRES: Finish() has not been called since the last call to Reset().
    // REQUIRES: key is larger than any previously added key
    void Add(const Slice& key, const Slice& value);

    // Finish building the block and return a slice that refers to the
    // block contents.  The returned slice will remain valid for the
    // lifetime of this builder or until Reset() is called.
    Slice Finish();

    // Returns an estimate of the current (uncompressed) size of the block
    // we are building.
    size_t CurrentSizeEstimate() const;

    // Return true iff no entries have been added since the last Reset()
    bool empty() const { return buffer_.empty(); }

private:
    const Options* options_;          // 默认配置
    std::string buffer_;              // Destination buffer
    std::vector<uint32_t> restarts_;  // Restart points(重启点的位置信息)
    int counter_;                     // Number of entries emitted since restart
    bool finished_;                   // Has Finish() been called?
    std::string last_key_;
};

block的数据格式如图所示，可以根据Add和Finish推导出来。

Add

添加数据时，如果没有超过重启点间隔，遍历key获取和last_key相同的前缀

const size_t min_length = std::min(last_key_piece.size(), key.size());
while ((shared < min_length) && (last_key_piece[shared] == key[shared])) {
	shared++;
}

如果超过了重启点间隔，那么记录重启点位置

restarts_.push_back(buffer_.size());
counter_ = 0;

重启点处理完了，就按照格式放入数据，add完整代码如下：

void BlockBuilder::Add(const Slice& key, const Slice& value) {
  Slice last_key_piece(last_key_);
  assert(!finished_);
  assert(counter_ <= options_->block_restart_interval);
  assert(buffer_.empty()  // No values yet?
         || options_->comparator->Compare(key, last_key_piece) > 0);
  size_t shared = 0;
  if (counter_ < options_->block_restart_interval) {
    // See how much sharing to do with previous string
    // 遍历相同前缀
    const size_t min_length = std::min(last_key_piece.size(), key.size());
    while ((shared < min_length) && (last_key_piece[shared] == key[shared])) {
      shared++;
    }
  } else {
    // Restart compression
    // 这条记录为重启点
    restarts_.push_back(buffer_.size());
    counter_ = 0;
  }
  const size_t non_shared = key.size() - shared;

  // Add "<shared><non_shared><value_size>" to buffer_
  // 添加一条记录
  PutVarint32(&buffer_, shared);
  PutVarint32(&buffer_, non_shared);
  PutVarint32(&buffer_, value.size());

  // Add string delta to buffer_ followed by value
  buffer_.append(key.data() + shared, non_shared);
  buffer_.append(value.data(), value.size());

  // Update state
  last_key_.resize(shared);
  last_key_.append(key.data() + shared, non_shared);
  assert(Slice(last_key_) == key);
  counter_++;
}

下面举例了在还原点内数据变化：

Finish

数据处理完成后，将重启点数据放入buffer_中，返回一个Slice结果。

Slice BlockBuilder::Finish() {
    // Append restart array
    for (size_t i = 0; i < restarts_.size(); i++) {
        PutFixed32(&buffer_, restarts_[i]);
    }
    PutFixed32(&buffer_, restarts_.size());
    finished_ = true;
    return Slice(buffer_);
}

CurrentSizeEstimate

获得当前block data的大小。

根据数据格式，数据包括key-value、重启点、重启点数量

size_t BlockBuilder::CurrentSizeEstimate() const {
    return (buffer_.size() +                       // Raw data buffer
            restarts_.size() * sizeof(uint32_t) +  // Restart array
            sizeof(uint32_t));                     // Restart array length
}

BlockBuilder、Reset

BlockBuilder::BlockBuilder(const Options* options)
    : options_(options), restarts_(), counter_(0), finished_(false) {
  assert(options->block_restart_interval >= 1);
  restarts_.push_back(0);  // First restart point is at offset 0
}

void BlockBuilder::Reset() {
  buffer_.clear();
  restarts_.clear();
  restarts_.push_back(0);  // First restart point is at offset 0
  counter_ = 0;
  finished_ = false;
  last_key_.clear();
}

Block

定义

block的定义很简单，除了构造和析构外，只提供size和迭代器的接口。

class Block {
public:
    // Initialize the block with the specified contents.
    explicit Block(const BlockContents& contents);

    Block(const Block&) = delete;
    Block& operator=(const Block&) = delete;

    ~Block();

    size_t size() const { return size_; }
    Iterator* NewIterator(const Comparator* comparator);

private:
    class Iter;

    uint32_t NumRestarts() const;

    const char* data_;
    size_t size_;
    uint32_t restart_offset_;  // Offset in data_ of restart array
    bool owned_;               // Block owns data_[]
};

构造函数

Block::Block(const BlockContents& contents)
    : data_(contents.data.data()),
	  size_(contents.data.size()),
	  owned_(contents.heap_allocated) {
    if (size_ < sizeof(uint32_t)) {
        size_ = 0;  // Error marker
    } else {
        // 表示一段size_大小的data_中可能存在的最大重启点个数
        size_t max_restarts_allowed = (size_ - sizeof(uint32_t)) / sizeof(uint32_t);
        if (NumRestarts() > max_restarts_allowed) {
            // The size is too small for NumRestarts()
            size_ = 0;
        } else {
            // 重启点数据的偏移量
            restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t);
        }
    }
}

BlockContents定义如下：

struct BlockContents {
    Slice data;           // Actual contents of data
    bool cachable;        // True iff data can be cached
    bool heap_allocated;  // True iff caller should delete[] data.data()
};

NumRestarts获取重启点个数，根据block格式，重启点个数保存在最后32位，实现如下：

inline uint32_t Block::NumRestarts() const {
    assert(size_ >= sizeof(uint32_t));
    // data_最后一个uint32_t字段表示重启点个数
    return DecodeFixed32(data_ + size_ - sizeof(uint32_t));
}

NewIterator

构建一个迭代器

Iterator* Block::NewIterator(const Comparator* comparator) {
  if (size_ < sizeof(uint32_t)) {
    return NewErrorIterator(Status::Corruption("bad block contents"));
  }
  const uint32_t num_restarts = NumRestarts();
  if (num_restarts == 0) {
    return NewEmptyIterator();
  } else {
    return new Iter(comparator, data_, restart_offset_, num_restarts);
  }
}

Block::Iter

构造

restarts、current初始化为重启点数据偏移量，num_restarts_初始化为重启点个数。

Iter(const Comparator* comparator, const char* data, uint32_t restarts,
     uint32_t num_restarts)
    : comparator_(comparator),
	  data_(data),
	  restarts_(restarts),
	  num_restarts_(num_restarts),
	  current_(restarts_),
	  restart_index_(num_restarts_) {
    assert(num_restarts_ > 0);
}

遍历

向后

主要根据block的格式往后遍历，实现逻辑比较清晰。

void Next() override {
	assert(Valid());
	ParseNextKey();
}
bool ParseNextKey() {
    current_ = NextEntryOffset();
    const char* p = data_ + current_;
    const char* limit = data_ + restarts_;  // Restarts come right after data

    // 异常
    if (p >= limit) {
        // No more entries to return.  Mark as invalid.
        current_ = restarts_;
        restart_index_ = num_restarts_;
        return false;
    }

    // Decode next entry
    uint32_t shared, non_shared, value_length;
    p = DecodeEntry(p, limit, &shared, &non_shared, &value_length);
    if (p == nullptr || key_.size() < shared) {
        CorruptionError();
        return false;
    } else {
        key_.resize(shared);
        key_.append(p, non_shared);
        value_ = Slice(p + non_shared, value_length);
        while (restart_index_ + 1 < num_restarts_ &&
               GetRestartPoint(restart_index_ + 1) < current_) {
            ++restart_index_;
        }
        return true;
    }
}

向前

先找到前一个重启点，然后顺序遍历两个重启点之间的数据，直到当前数据尾部等于之前头部。

void Prev() override {
	assert(Valid());

    // Scan backwards to a restart point before current_
    const uint32_t original = current_;
    // 找到前一个重启点
    while (GetRestartPoint(restart_index_) >= original) {
        if (restart_index_ == 0) {
            // No more entries
            current_ = restarts_;
            restart_index_ = num_restarts_;
            return;
        }
        restart_index_--;
    }

    SeekToRestartPoint(restart_index_);
    // 遍历，直到当前数据尾部等于之前头部
    do {
    // Loop until end of current entry hits the start of original entry
    } while (ParseNextKey() && NextEntryOffset() < original);
}

uint32_t GetRestartPoint(uint32_t index) {
    assert(index < num_restarts_);
    return DecodeFixed32(data_ + restarts_ + index * sizeof(uint32_t));
}

void SeekToRestartPoint(uint32_t index) {
    key_.clear();
    restart_index_ = index;
    // current_ will be fixed by ParseNextKey();

    // ParseNextKey() starts at the end of value_, so set value_ accordingly
    uint32_t offset = GetRestartPoint(index);
    value_ = Slice(data_ + offset, 0);
}

查找

二分查找，查找第一条key大于target的记录。

void Seek(const Slice& target) override {
    // Binary search in restart array to find the last restart point
    // with a key < target
    uint32_t left = 0;
    uint32_t right = num_restarts_ - 1;
    int current_key_compare = 0;

    // 1.根据当前key缩小搜索范围
    if (Valid()) {
        // If we're already scanning, use the current position as a starting
        // point. This is beneficial if the key we're seeking to is ahead of the
        // current position.
        current_key_compare = Compare(key_, target);
        if (current_key_compare < 0) {
            // key_ is smaller than target
            left = restart_index_;
        } else if (current_key_compare > 0) {
            right = restart_index_;
        } else {
            // We're seeking to the key we're already at.
            return;
        }
    }
	// 2.二分查找，找到相应重启点位置
    while (left < right) {
        uint32_t mid = (left + right + 1) / 2;
        uint32_t region_offset = GetRestartPoint(mid);
        uint32_t shared, non_shared, value_length;
        const char* key_ptr =
            DecodeEntry(data_ + region_offset, data_ + restarts_, &shared,
                        &non_shared, &value_length);
        if (key_ptr == nullptr || (shared != 0)) {
            CorruptionError();
            return;
        }
        Slice mid_key(key_ptr, non_shared);
        if (Compare(mid_key, target) < 0) {
            // Key at "mid" is smaller than "target".  Therefore all
            // blocks before "mid" are uninteresting.
            left = mid;
        } else {
            // Key at "mid" is >= "target".  Therefore all blocks at or
            // after "mid" are uninteresting.
            right = mid - 1;
        }
    }

    // We might be able to use our current position within the restart block.
    // This is true if we determined the key we desire is in the current block
    // and is after than the current key.
    assert(current_key_compare == 0 || Valid());
    bool skip_seek = left == restart_index_ && current_key_compare < 0;
    if (!skip_seek) {
        SeekToRestartPoint(left);
    }
    // 3.在还原点内线性搜索
    // Linear search (within restart block) for first key >= target
    while (true) {
        if (!ParseNextKey()) {
            return;
        }
        if (Compare(key_, target) >= 0) {
            return;
        }
    }
}