jemalloc 深入分析 之radix tree 基数树

为了更好的阅读效果，推荐下载pdf文档：
详细文章请参考：《jemalloc 深入分析》
https://github.com/everschen/tools/blob/master/DOC/Jemalloc.pdf
https://download.csdn.net/download/ip5108/10941278

2.3. radix tree 基数树

• This radix tree implementation is tailored to the singular purpose of
• associating metadata with chunks that are currently owned by jemalloc.
  对于长整型数据的映射。怎样解决Hash冲突和Hash表大小的设计是一个非常头疼的问题。radix树就是针对这样的稀疏的长整型数据查找，能高速且节省空间地完毕映射。借助于radix树，我们能够实现对于长整型数据类型的路由。
  利用radix树能够依据一个长整型（比方一个长ID）高速查找到其相应的对象指针。这比用hash映射来的简单，也更节省空间，使用hash映射hash函数难以设计，不恰当的hash函数可能增大冲突，或浪费空间。
  radix tree是一种多叉搜索树。树的叶子结点是实际的数据条目。每一个结点有一个固定的、2^n指针指向子结点（每一个指针称为槽slot，n为划分的基的大小）（如果n=2，就有4个指针指向子节点）。
  radix树为稀疏树提供了有效的存储，取代固定尺寸数组提供了键值到指针的高速查找。

2.3.1. radix tree的level和对应的subtree结构

2.3.2. radix tree头文件定义
struct rtree_node_elm_s {
union {
void *pun;
rtree_node_elm_t *child;
extent_node_t val;
};
};
struct rtree_level_s {
/

• A non-NULL subtree points to a subtree rooted along the hypothetical
• path to the leaf node corresponding to key 0. Depending on what keys
• have been used to store to the tree, an arbitrary combination of
  Jemalloc 深入分析
  Copyright 2013 Spreadtrum Communications Inc. 17
• subtree pointers may remain NULL.
• Suppose keys comprise 48 bits, and LG_RTREE_BITS_PER_LEVEL is 4.
• This results in a 3-level tree, and the leftmost leaf can be directly
• accessed via subtrees[2], the subtree prefixed by 0x0000 (excluding
• 0x00000000) can be accessed via subtrees[1], and the remainder of the
• tree can be accessed via subtrees[0].
• levels[0] : [ | 0x0001******** | 0x0002******** | …]
• levels[1] : [ | 0x00000001**** | 0x00000002**** | … ]
• levels[2] : [val(0x000000000000) | val(0x000000000001) | …]
• This has practical implications on x64, which currently uses only the
• lower 47 bits of virtual address space in userland, thus leaving
• subtrees[0] unused and avoiding a level of tree traversal.
  */
  union {
  void *subtree_pun;
  rtree_node_elm_t subtree;
  };
  / Number of key bits distinguished by this level. /
  unsigned bits;
  /
• Cumulative number of key bits distinguished by traversing to
• corresponding tree level.
  */
  unsigned cumbits;
  };
  struct rtree_s {
  rtree_node_alloc_t *alloc;
  rtree_node_dalloc_t dalloc;
  unsigned height;
  /
  Jemalloc 深入分析
  Copyright 2013 Spreadtrum Communications Inc. 18
• Precomputed table used to convert from the number of leading 0 key
• bits to which subtree level to start at.
  */
  unsigned start_level[RTREE_HEIGHT_MAX];
  rtree_level_t levels[RTREE_HEIGHT_MAX];
  };

2.3.3. radix tree在jemalloc的实现
JEMALLOC_ALWAYS_INLINE unsigned
rtree_start_level(rtree_t *rtree, uintptr_t key)
{
unsigned start_level;
if (unlikely(key == 0))
return (rtree->height - 1);
start_level = rtree->start_level[lg_floor(key) >>
LG_RTREE_BITS_PER_LEVEL];
assert(start_level < rtree->height);
return (start_level);
}
对于key最高位1的位置（从0开始计数）如果
key范围：0 - 2^16-1, lg_floor(key) =0->15 , i=0, start_level=2,
key范围：2^16 - 2^32-1, lg_floor(key) =16->31, i=1, start_level=2, key范围：2^32 - 2^48-1, lg_floor(key) =32->47, i=2, start_level=1,
key范围：2^48 - 2^64-1, lg_floor(key) =48->63, i=3, start_level=0,
这个start_level，对于可以2^32 - 2^48-1, lg_floor(key) =32->47，因为高16位为0，所以start_level直接从1开始。
(gdb) p je_chunks_rtree
$162 = {alloc = 0x7247636b60 <chunks_rtree_node_alloc>, dalloc = 0x0, height = 3, start_level = {2, 2, 1, 0}, levels = {
{{subtree_pun = 0x0, subtree = 0x0}, bits = 16, cumbits = 16}, // {{subtree_pun = 0x7246e0bc00, subtree = 0x7246e0bc00}, bits = 16,cumbits = 32}, {{subtree_pun = 0x0, subtree = 0x0}, bits = 11,cumbits = 43}, //
{{subtree_pun = 0x0, subtree = 0x0}, bits = 0,cumbits = 0}//
}}
Jemalloc 深入分析
Copyright 2013 Spreadtrum Communications Inc. 19
从这里可以看出，当前只有level=1的，key范围2^32 - 2^48-1, lg_floor(key) =32->47,的chunk存在。Small和large的chunk也是存这里吗？是的，后面通过gdb调试可以查看到所有的chunk地址。
rtree_subkey(rtree_t rtree, uintptr_t key, unsigned level)
{
return ((key >> ((ZU(1) << (LG_SIZEOF_PTR+3)) -
rtree->levels[level].cumbits)) & ((ZU(1) <<
rtree->levels[level].bits) - 1));
}
如果level=0，rtree_subkey返回key前16bits的值，值范围[0,65535]
如果level=1，rtree_subkey返回key高位第二个16bits的值，值范围[0,65535]
如果level=2，rtree_subkey返回key后续11bits的值，值范围[0,2047]
rtree_new(&chunks_rtree, (unsigned)((ZU(1) << (LG_SIZEOF_PTR+3)) -
opt_lg_chunk), chunks_rtree_node_alloc, NULL)
((ZU(1) << (LG_SIZEOF_PTR+3)) - opt_lg_chunk)=2^6-21=43，21是chunk大小2M的后21位全0，所以不用作为key，真正作为key的是前64-21=43位chunk的地址。
/ Only the most significant bits of keys passed to rtree_[gs]et() are used. */
bool
rtree_new(rtree_t *rtree, unsigned bits, rtree_node_alloc_t *alloc,
rtree_node_dalloc_t dalloc)
{
unsigned bits_in_leaf, height, i;
assert(RTREE_HEIGHT_MAX == ((ZU(1) << (LG_SIZEOF_PTR+3)) /
RTREE_BITS_PER_LEVEL));
assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3));
bits_in_leaf = (bits % RTREE_BITS_PER_LEVEL) == 0 ? RTREE_BITS_PER_LEVEL: (bits % RTREE_BITS_PER_LEVEL); bits_in_leaf=43%16==0?16:(43%16) bits_in_leaf =11
if (bits > bits_in_leaf) {
Jemalloc 深入分析
Copyright 2013 Spreadtrum Communications Inc. 20
height = 1 + (bits - bits_in_leaf) / RTREE_BITS_PER_LEVEL; height=1+(43-11)/16=3
if ((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf != bits)
height++;
} else
height = 1;
assert((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf == bits);
rtree->alloc = alloc;
rtree->dalloc = dalloc;
rtree->height = height;
/ Root level. /
rtree->levels[0].subtree = NULL;
rtree->levels[0].bits = (height > 1) ? RTREE_BITS_PER_LEVEL : bits_in_leaf;//=16
rtree->levels[0].cumbits = rtree->levels[0].bits;
/ Interior levels. /
for (i = 1; i < height-1; i++) {
rtree->levels[i].subtree = NULL; rtree->levels[i].bits = RTREE_BITS_PER_LEVEL;//=16
rtree->levels[i].cumbits = rtree->levels[i-1].cumbits + RTREE_BITS_PER_LEVEL;//=32
}
/ Leaf level. /
if (height > 1) {
rtree->levels[height-1].subtree = NULL; rtree->levels[height-1].bits = bits_in_leaf;//=11 rtree->levels[height-1].cumbits = bits;//=43
}
/ Compute lookup table to be used by rtree_start_level(). */
for (i = 0; i < RTREE_HEIGHT_MAX; i++) {
rtree->start_level[i] = hmin(RTREE_HEIGHT_MAX - 1 - i, height - 1);//2,2,1,0
}
Jemalloc 深入分析
Copyright 2013 Spreadtrum Communications Inc. 21
return (false);
}
JEMALLOC_ALWAYS_INLINE rtree_node_elm_t *
rtree_subtree_read(rtree_t *rtree, unsigned level, bool dependent)
{
rtree_node_elm_t *subtree;
subtree = rtree_subtree_tryread(rtree, level, dependent);
if (!dependent && unlikely(!rtree_node_valid(subtree)))
subtree = rtree_subtree_read_hard(rtree, level);
assert(!dependent || subtree != NULL);
return (subtree);
}
得到start_level的subtree，如果没有，则初始化2^bits个subtree地址为首地址的rtree_node_elm_t *指针。
JEMALLOC_INLINE bool
rtree_set(rtree_t *rtree, uintptr_t key, const extent_node_t *val)
{
uintptr_t subkey;
unsigned i, start_level;
rtree_node_elm_t *node, child;
start_level = rtree_start_level(rtree, key); //由key的最高位1的位置，得到下面开始的start_level，再由这个得到node的地址
node = rtree_subtree_read(rtree, start_level, false);
if (node == NULL)
return (true);
for (i = start_level; /**/; i++, node = child) {
subkey = rtree_subkey(rtree, key, i);
if (i == rtree->height - 1) {
/

• node is a leaf, so it contains values rather than
• child pointers.
  Jemalloc 深入分析
  Copyright 2013 Spreadtrum Communications Inc. 22
  */ //如果已经是叶子节点，则把val值存入对应的subkey的位置
  rtree_val_write(rtree, &node[subkey], val);
  return (false);
  }
  assert(i + 1 < rtree->height); //如果还不是叶子节点，由subkey得到下一level的node地址
  child = rtree_child_read(rtree, &node[subkey], i, false);
  if (child == NULL)
  return (true);
  }
  not_reached();
  }
  JEMALLOC_ALWAYS_INLINE rtree_node_elm_t *
  rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm, unsigned level,
  bool dependent)
  {
  rtree_node_elm_t *child;
  child = rtree_child_tryread(elm, dependent);
  if (!dependent && unlikely(!rtree_node_valid(child))) child = rtree_child_read_hard(rtree, elm, level);
  assert(!dependent || child != NULL);
  return (child);
  }
  读取child值，如果child为空，则创建该level对应数量的rtree_node_elm_t *指针。

2.3.4. radix tree gdb 数据
(gdb) p je_chunks_rtree
$294 = {alloc = 0x6f89b7950c <chunks_rtree_node_alloc>, dalloc = 0x0,
height = 3, start_level = {2, 2, 1, 0}, levels = {
{{subtree_pun = 0x0,subtree = 0x0}, bits = 16, cumbits = 16},
{{subtree_pun = 0x6f8940bc00, subtree = 0x6f8940bc00}, bits = 16,cumbits = 32},
{{subtree_pun = 0x0, subtree = 0x0}, bits = 11,cumbits = 43},
Jemalloc 深入分析
Copyright 2013 Spreadtrum Communications Inc. 23
{{subtree_pun = 0x0, subtree = 0x0}, bits = 0,cumbits = 0}
}}
查看level=1下的所有的subtree值，这里只有一个，在subkey=111的位置：
(gdb) p *je_chunks_rtree->levels[1]->subtree@65536
$300 = {{{pun = 0x0, child = 0x0, val = 0x0}} <repeats 111 times>, {{
pun = 0x6f8948bc00, child = 0x6f8948bc00, val = 0x6f8948bc00}}, {{
pun = 0x0, child = 0x0, val = 0x0}} <repeats 65424 times>}
查看subkey=111的指向的child的元素，总共有2048个，有效的实际存储为7个。
(gdb) p *je_chunks_rtree->levels[1]->subtree[111]->child@2048
$313 = {{{pun = 0x0, child = 0x0, val = 0x0}} <repeats 1060 times>, {{ pun = 0x6f88328000, child = 0x6f88328000, val = 0x6f88328000}}, {{
pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x0, child = 0x0,
val = 0x0}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x6f8829e380, child = 0x6f8829e380, val = 0x6f8829e380}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{
pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x6f88234500, child = 0x6f88234500, val = 0x6f88234500}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{
pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x6f8829e680, child = 0x6f8829e680, val = 0x6f8829e680}}, {{pun = 0x0, child = 0x0, val = 0x0}} <repeats 11 times>, {{pun = 0x6f87a00000, child = 0x6f87a00000, val = 0x6f87a00000}}, {{pun = 0x0, child = 0x0,
val = 0x0}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x6f88200000, child = 0x6f88200000, val = 0x6f88200000}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{
pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x0, child = 0x0,
val = 0x0}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x0,
child = 0x0, val = 0x0}}, {{pun = 0x0, child = 0x0, val = 0x0}}, {{ pun = 0x0, child = 0x0, val = 0x0}}, {{pun = 0x6f89200000, child = 0x6f89200000, val = 0x6f89200000}}, {{pun = 0x0, child = 0x0,
val = 0x0}} <repeats 950 times>}
这个时候查看系统的chunk数量和地址：
Jemalloc 深入分析
Copyright 2013 Spreadtrum Communications Inc. 24
(gdb) p (*je_arenas[0])->achunks $314 = {qlh_first = 0x6f89200000}
(gdb) p (*je_arenas[0])->achunks->qlh_first
$315 = (extent_node_t *) 0x6f89200000
(gdb) p (*je_arenas[0])->achunks->qlh_first->ql_link.qre_next $316 = (extent_node_t *) 0x6f87a00000
qlh_first->ql_link.qre_next->ql_link.qre_next
$317 = (extent_node_t *) 0x6f89200000
(gdb) p (*je_arenas[1])->achunks $319 = {qlh_first = 0x6f88200000}
(gdb) p (*je_arenas[1])->achunks->qlh_first->ql_link.qre_next
$320 = (extent_node_t *) 0x6f88200000
再查看huge的chunk数量和地址：
(gdb) p (*je_arenas[1])->huge->qlh_first $333 = (extent_node_t *) 0x6f8829e680
(gdb) p (*je_arenas[1])->huge->qlh_first->ql_link.qre_nex
There is no member named qre_nex.
(gdb) p (*je_arenas[1])->huge->qlh_first->ql_link.qre_next $334 = (extent_node_t *) 0x6f88234500
(gdb) p (*je_arenas[1])->huge->qlh_first->ql_link.qre_next->ql_link.qre_next $335 = (extent_node_t *) 0x6f8829e380
qlh_first->ql_link.qre_next->ql_link.qre_next->ql_link.qre_next $336 = (extent_node_t *) 0x6f88328000
<l_link.qre_next->ql_link.qre_next->ql_link.qre_next->ql_link.qre_next
$337 = (extent_node_t *) 0x6f8829e680
刚好是上面看到的对应的7个chunk地址，另外还可以看出来，huge的extent_node_t *地址因为不能复用chunk的起始地址，所以不是5个0结尾的，而系统分配的其他chunk地址的extent_node_t *则是5个0结尾的，也就是chunk的首地址。

详细文章请参考：《jemalloc 深入分析》
https://github.com/everschen/tools/blob/master/DOC/Jemalloc.pdf
https://download.csdn.net/download/ip5108/10941278