简介
将上文提到的内存池,缓存,随机化跳表的技术集合起来
定义
ValueStruct
由于一个 Value 可能不仅仅是一个字符串,我们用一个结构体来管理
type ValueStruct struct {
Meta byte
Value []byte
ExpiresAt uint64 //过期时间
Version uint64 //版本号
}
Entry
kv 引擎写入类
type Entry struct {
Key []byte
Value ValueStruct
}
Arena
内存池
type Arena struct {
offset uint32 //偏移地址
buf []byte //数据
}
Node
跳表节点
type node struct {
KeyBlock uint64 // offset,sz
valueBlock uint64
h int //存储节点高度
levels [MaxHeight]uint32 // 存储跳表节点在 Arena 中的偏移地址
}
SkipList
跳表本体
type SkipList struct {
height int32 //[0, MaxHeight]
arena *Arena
headOffset uint32 //存储head的地址
size int32 //存储节点数量
OnClose func() //删除跳表时调用
}
接口设计
ValueStruct
编码解码
对 valueStruct 需要进行编码和解码操作
func calcVarIntSize(x uint64) (sz uint32) {//计算变长编码长度
for {
sz++
x >>= 7
if x == 0 {
break
}
}
return sz
}
func (this *ValueStruct) CalcEncodedSize() uint32 {
sz := len(this.Value) + 1 // + Meta
return uint32(sz) + calcVarIntSize(this.ExpiresTime)
}
func (this *ValueStruct) EncodeIn(buf []byte) uint32 { // 返回实际编码的大小
buf[0] = this.Meta
sz := binary.PutUvarint(buf[1:], this.ExpiresTime) //变长编码
n := copy(buf[1+sz:], this.Value)
return uint32(1 + sz + n)
}
func (this *ValueStruct) DecodeFrom(buf []byte) {
this.Meta = buf[0]
var sz int
this.ExpiresTime, sz = binary.Uvarint(buf[1:])
this.Value = buf[1+sz:]
}
Entry
NewEntry(key, val []byte) *Entry
Entry
存储 kv 对
创建一个 包含 k
, v
的 Entry
func NewEntry(key, val []byte) *Entry {
return &Entry{
Key: key,
Value: ValueStruct{Value: val},
}
}
Arena
func newArena(n int64) *Arena
创建一个内存大小为 n n n 的内存池
func newArena(n int64) *Arena {
return &Arena{
offset: 0,
buf: make([]byte, n),
}
}
func (s *Arena) allocate(n uint32) uint32
分配一块大小为
n
n
n 内存,返回内存块在 Arena
中的起始地址
func (s *Arena) allocate(n uint32) uint32 { //返回偏移地址
offset := atomic.AddUint32(&s.offset, n)
lenth := len(s.buf)
if lenth-int(offset) < MaxNodeSize { //空间不足翻倍
grow := lenth
if grow > MaxBufSize {
grow = MaxBufSize
}
if uint32(grow) < n { // 防止一次扩展不够
grow = int(n)
}
newBuf := make([]byte, lenth+grow)
copy(newBuf, s.buf)
s.buf = newBuf
}
return offset - n
}
为 SkipList 分配内存
注意变长编码,当时调代码被卡了3个小时
func (s *Arena) size() int64 {
return int64(atomic.LoadUint32(&s.offset))
}
func (s *Arena) putNode(height int) uint32 {
//baseSize := uint32(unsafe.Sizeof(uint32(0))) //取得32/64位机器的基本数据大小
unusedSize := uint32(MaxHeight-height-1) * BaseSize
sz := uint32(MaxNodeSize) - unusedSize
sz += BaseSize - (sz-1)%BaseSize - 1 //向前内存对齐
offset := s.allocate(sz)
return offset
}
func (s *Arena) putVal(v *ValueStruct) uint32 {
sz := v.CalcEncodedSize()
sz += BaseSize - (sz-1)%BaseSize - 1
offset := s.allocate(sz)
v.EncodeIn(s.buf[offset:])
return offset
}
func (s *Arena) putKey(key []byte) uint32 {
sz := uint32(len(key))
sz += BaseSize - (sz-1)%BaseSize - 1
offset := s.allocate(sz)
copy(s.buf[offset:], key)
return offset
}
内存查找
接收地址返回实例指针
func (this *Arena) GetNode(offset uint32) *node {
return (*node)(unsafe.Pointer(&this.buf[offset]))
}
func (this *Arena) GetKey(offset, sz uint32) []byte {
return this.buf[offset : offset+sz]
}
func (this *Arena) GetValue(offset, sz uint32) *ValueStruct {
value := new(ValueStruct)
value.DecodeFrom(this.buf[offset : offset+sz])
return value
}
node
新建节点
需要在内存池中分配内存
func newNode(arena *Arena, key []byte, val ValueStruct, height int) (*node, uint32) {
nodeOffset := arena.putNode(height)
keyOffset := arena.putKey(key)
keySize := uint32(len(key))
valOffset := arena.putVal(val)
valSize := val.CalcEncodedSize()
nd := arena.GetNode(nodeOffset)
nd.KeyBlock = encodeKey(keyOffset, keySize)
nd.valueBlock = encodeValue(valOffset, valSize)
nd.h = height
return nd, nodeOffset
}
SkipList
NewSkipList(arenaSize int64) *SkipList
创建一个内存池大小为 arenaSize
的跳表
注意创建表头
func NewSkipList(arenaSize int64) *SkipList {
arena := newArena(arenaSize)
_, headOffset := newNode(arena, nil, &ValueStruct{}, MaxHeight-1)
return &SkipList{
headOffset: headOffset,
arena: arena,
size: 1,
}
}
Insert(e *Entry)
在跳表中插入一个 Entry
注意可以覆盖掉原有 Value
func checkUp() bool {
return rand.Intn(2) == 1
}
func (list *SkipList) Insert(e *Entry) {
pre := list.arena.GetNode(list.headOffset)
hashKey := calcHashKey(e.Key)
var preElemHeaders [MaxHeight]*node //注意上界,可能需要调整或取min
for i := MaxHeight - 1; i >= 0; i-- {
preElemHeaders[i] = pre
for nxt := pre.levels[i]; nxt != 0; nxt = pre.levels[i] {
now := list.arena.GetNode(nxt)
res := cmp(hashKey, e.Key, now, list.arena)
if res == 0 {
valOffset := list.arena.putVal(e.Value)
valSize := e.Value.CalcEncodedSize()
now.valueBlock = encodeValue(valOffset, valSize)
}
if res == -1 {
preElemHeaders[i] = pre
break
}
pre = now
}
}
len := 0
for checkUp() {
len++
}
element, elementOffset := newNode(list.arena, e.Key, e.Value, len)
for i := 0; i <= len; i++ {
element.levels[i] = preElemHeaders[i].levels[i]
preElemHeaders[i].levels[i] = elementOffset
}
}
Find(key []byte) ValueStruct
在跳表中查找,返回一个 ValueStruct
func calcHashKey(key []byte) uint64 {
len := len(key)
if len > 8 {
len = 8
}
var res uint64
for i := 0; i < len; i++ {
res |= uint64(key[i]) << (64 - (i+1)*8)
}
return res
}
func cmp(hashKey uint64, key []byte, b *node, arena *Arena) int {
if hashKey == b.HashKey {
bkeyOffset, bkeySize := decodeKey(b.KeyBlock)
return bytes.Compare(key, arena.GetKey(bkeyOffset, bkeySize))
}
if hashKey < b.HashKey {
return -1
} else {
return 1
}
}
func (list *SkipList) Find(key []byte) *ValueStruct {
pre := list.arena.GetNode(list.headOffset)
hashKey := calcHashKey(key)
for i := MaxHeight - 1; i >= 0; i-- {
for nxt := pre.levels[i]; nxt != 0; nxt = pre.levels[i] {
now := list.arena.GetNode(nxt)
res := cmp(hashKey, key, now, list.arena)
if res == 0 {
return list.arena.GetValue(decodeValue(now.valueBlock))
} else {
if res == -1 {
break
}
}
pre = now
}
}
return nil
}