工业化跳表实现

简介

将上文提到的内存池，缓存，随机化跳表的技术集合起来

定义

ValueStruct

由于一个 Value 可能不仅仅是一个字符串，我们用一个结构体来管理

type ValueStruct struct {
	Meta      byte
	Value     []byte
	ExpiresAt uint64 //过期时间
	Version   uint64 //版本号
}

Entry

kv 引擎写入类

type Entry struct {
	Key []byte
	Value ValueStruct
}

Arena

内存池

type Arena struct {
	offset uint32 //偏移地址
	buf    []byte //数据
}

Node

跳表节点

type node struct {
	KeyBlock   uint64 // offset,sz
	valueBlock uint64
	h          int               //存储节点高度
	levels     [MaxHeight]uint32 // 存储跳表节点在 Arena 中的偏移地址
}

SkipList

跳表本体

type SkipList struct {
	height     int32 //[0, MaxHeight]
	arena      *Arena
	headOffset uint32 //存储head的地址
	size       int32  //存储节点数量
	OnClose    func() //删除跳表时调用
}

接口设计

ValueStruct

编码解码

对 valueStruct 需要进行编码和解码操作

func calcVarIntSize(x uint64) (sz uint32) {//计算变长编码长度
	for {
		sz++
		x >>= 7
		if x == 0 {
			break
		}
	}
	return sz
}
func (this *ValueStruct) CalcEncodedSize() uint32 {
	sz := len(this.Value) + 1 // + Meta
	return uint32(sz) + calcVarIntSize(this.ExpiresTime)
}
func (this *ValueStruct) EncodeIn(buf []byte) uint32 { // 返回实际编码的大小
	buf[0] = this.Meta
	sz := binary.PutUvarint(buf[1:], this.ExpiresTime) //变长编码
	n := copy(buf[1+sz:], this.Value)
	return uint32(1 + sz + n)
}
func (this *ValueStruct) DecodeFrom(buf []byte) {
	this.Meta = buf[0]
	var sz int
	this.ExpiresTime, sz = binary.Uvarint(buf[1:])
	this.Value = buf[1+sz:]
}

Entry

NewEntry(key, val []byte) *Entry

Entry 存储 kv 对
创建一个 包含 k, v 的 Entry

func NewEntry(key, val []byte) *Entry {
	return &Entry{
		Key:   key,
		Value: ValueStruct{Value: val},
	}
}

Arena

func newArena(n int64) *Arena

创建一个内存大小为 $n$ 的内存池

func newArena(n int64) *Arena {
	return &Arena{
		offset: 0,
		buf:    make([]byte, n),
	}
}

func (s *Arena) allocate(n uint32) uint32

分配一块大小为 $n$ 内存，返回内存块在 Arena 中的起始地址

func (s *Arena) allocate(n uint32) uint32 { //返回偏移地址
	offset := atomic.AddUint32(&s.offset, n)
	lenth := len(s.buf)
	if lenth-int(offset) < MaxNodeSize { //空间不足翻倍
		grow := lenth
		if grow > MaxBufSize {
			grow = MaxBufSize
		}
		if uint32(grow) < n { // 防止一次扩展不够
			grow = int(n)
		}
		newBuf := make([]byte, lenth+grow)
		copy(newBuf, s.buf)
		s.buf = newBuf
	}
	return offset - n
}

为 SkipList 分配内存

注意变长编码，当时调代码被卡了3个小时

func (s *Arena) size() int64 {
	return int64(atomic.LoadUint32(&s.offset))
}
func (s *Arena) putNode(height int) uint32 {
	//baseSize := uint32(unsafe.Sizeof(uint32(0))) //取得32/64位机器的基本数据大小
	unusedSize := uint32(MaxHeight-height-1) * BaseSize
	sz := uint32(MaxNodeSize) - unusedSize
	sz += BaseSize - (sz-1)%BaseSize - 1 //向前内存对齐
	offset := s.allocate(sz)
	return offset
}
func (s *Arena) putVal(v *ValueStruct) uint32 {
	sz := v.CalcEncodedSize()
	sz += BaseSize - (sz-1)%BaseSize - 1
	offset := s.allocate(sz)
	v.EncodeIn(s.buf[offset:])
	return offset
}
func (s *Arena) putKey(key []byte) uint32 {
	sz := uint32(len(key))
	sz += BaseSize - (sz-1)%BaseSize - 1
	offset := s.allocate(sz)
	copy(s.buf[offset:], key)
	return offset
}

内存查找

接收地址返回实例指针

func (this *Arena) GetNode(offset uint32) *node {
	return (*node)(unsafe.Pointer(&this.buf[offset]))
}
func (this *Arena) GetKey(offset, sz uint32) []byte {
	return this.buf[offset : offset+sz]
}
func (this *Arena) GetValue(offset, sz uint32) *ValueStruct {
	value := new(ValueStruct)
	value.DecodeFrom(this.buf[offset : offset+sz])
	return value
}

node

新建节点

需要在内存池中分配内存

func newNode(arena *Arena, key []byte, val ValueStruct, height int) (*node, uint32) {
	nodeOffset := arena.putNode(height)
	keyOffset := arena.putKey(key)
	keySize := uint32(len(key))
	valOffset := arena.putVal(val)
	valSize := val.CalcEncodedSize()
	nd := arena.GetNode(nodeOffset)
	nd.KeyBlock = encodeKey(keyOffset, keySize)
	nd.valueBlock = encodeValue(valOffset, valSize)
	nd.h = height
	return nd, nodeOffset
}

SkipList

NewSkipList(arenaSize int64) *SkipList

创建一个内存池大小为 arenaSize 的跳表
注意创建表头

func NewSkipList(arenaSize int64) *SkipList {
	arena := newArena(arenaSize)
	_, headOffset := newNode(arena, nil, &ValueStruct{}, MaxHeight-1)
	return &SkipList{
		headOffset: headOffset,
		arena:      arena,
		size:       1,
	}
}

Insert(e *Entry)

在跳表中插入一个 Entry
注意可以覆盖掉原有 Value

func checkUp() bool {
	return rand.Intn(2) == 1
}
func (list *SkipList) Insert(e *Entry) {
	pre := list.arena.GetNode(list.headOffset)
	hashKey := calcHashKey(e.Key)
	var preElemHeaders [MaxHeight]*node //注意上界，可能需要调整或取min
	for i := MaxHeight - 1; i >= 0; i-- {
		preElemHeaders[i] = pre
		for nxt := pre.levels[i]; nxt != 0; nxt = pre.levels[i] {
			now := list.arena.GetNode(nxt)
			res := cmp(hashKey, e.Key, now, list.arena)
			if res == 0 {
				valOffset := list.arena.putVal(e.Value)
				valSize := e.Value.CalcEncodedSize()
				now.valueBlock = encodeValue(valOffset, valSize)
			}
			if res == -1 {
				preElemHeaders[i] = pre
				break
			}
			pre = now
		}
	}
	len := 0
	for checkUp() {
		len++
	}
	element, elementOffset := newNode(list.arena, e.Key, e.Value, len)
	for i := 0; i <= len; i++ {
		element.levels[i] = preElemHeaders[i].levels[i]
		preElemHeaders[i].levels[i] = elementOffset
	}
}

Find(key []byte) ValueStruct

在跳表中查找，返回一个 ValueStruct

func calcHashKey(key []byte) uint64 {
	len := len(key)
	if len > 8 {
		len = 8
	}
	var res uint64
	for i := 0; i < len; i++ {
		res |= uint64(key[i]) << (64 - (i+1)*8)
	}
	return res
}
func cmp(hashKey uint64, key []byte, b *node, arena *Arena) int {
	if hashKey == b.HashKey {
		bkeyOffset, bkeySize := decodeKey(b.KeyBlock)
		return bytes.Compare(key, arena.GetKey(bkeyOffset, bkeySize))
	}
	if hashKey < b.HashKey {
		return -1
	} else {
		return 1
	}
}
func (list *SkipList) Find(key []byte) *ValueStruct {
	pre := list.arena.GetNode(list.headOffset)
	hashKey := calcHashKey(key)
	for i := MaxHeight - 1; i >= 0; i-- {
		for nxt := pre.levels[i]; nxt != 0; nxt = pre.levels[i] {
			now := list.arena.GetNode(nxt)
			res := cmp(hashKey, key, now, list.arena)
			if res == 0 {
				return list.arena.GetValue(decodeValue(now.valueBlock))
			} else {
				if res == -1 {
					break
				}
			}
			pre = now
		}
	}
	return nil
}