memory managerment slab allocator(2)

怎样调试memory出现的错误：

red zoning/ object poisoning/ caller到底在哪里？

per_cpu变量使用的memory /object就是 slab的object？

kmalloc 分到的memory到底是怎样分布的？

测试代码：

create_cache and allocate an object

/*****************************************************************************/

static void init_my_cache(void)
{
   my_cachep = kmem_cache_create(
                  "my_cache",        /* Name */
                  32,                   /* Object Size */
                  0,                    /* Alignment */
                  SLAB_HWCACHE_ALIGN,   /* Flags */
                  NULL );                  /* Constructor/Deconstructor */
   return;
}

int slab_test(void)
{
  void *object;

  pr_err("slab_test: Cache name is %s\n", my_cachep->name);
  pr_err("slab_test: Cache object size is %d\n", kmem_cache_size(my_cachep));
  object = kmem_cache_alloc(my_cachep,GFP_KERNEL );
  if (object){
    pr_err("slab_test: get an object %p\n", object);
  }
  return 0;
}

[    4.076523:1] slab_test: Cache name is my_cache
[    4.081039:1] slab_test: Cache object size is 32
[    4.085680:1] slab_test: get an object ee29e478

/*****************************************************************************************/

从dump分析cache的创建使用过程

crash> kmem -s
CACHE    NAME                 OBJSIZE  ALLOCATED     TOTAL  SLABS  SSIZE
crash> kmem -s | grep my_cache
ee29da00 my_cache                  56          1        67      1     4k

结构体kmem_cache

crash> kmem_cache ee29da00
struct kmem_cache {
  batchcount = 16,
  limit = 32,
  shared = 8,
  buffer_size = 56, /*这里的buffer_size == 56,为什么不是32? 还有24个字节的位置，放的是什么？*/
  reciprocal_buffer_size = 76695845,
  flags = 76800,
  num = 67,
  gfporder = 0,
  gfpflags = 0,
  colour = 1,
  colour_off = 32,
  slabp_cache = 0x0,
  slab_size = 296,
  dflags = 0,
  ctor = 0x0,
  name = 0xc064221b "my_cache",
  next = {
    next = 0xee278ac0,
    prev = 0xee29d9a0
  },
  num_active = 1,
  num_allocations = 16,
  high_mark = 16,
  grown = 1,
  reaped = 0,
  errors = 0,
  max_freeable = 0,
  node_allocs = 0,
  node_frees = 0,
  node_overflow = 0,
  allochit = {
    counter = 0
  },
  allocmiss = {
    counter = 1
  },
  freehit = {
    counter = 0
  },
  freemiss = {
    counter = 0
  },
  obj_offset = 8,
  obj_size = 32,
  nodelists = 0xee29da94,
  array = {0xee1afe60, 0xee1afda0}
}

crash> kmem_cache
struct kmem_cache {
    unsigned int batchcount;
    unsigned int limit;
    unsigned int shared;
    unsigned int buffer_size;
    u32 reciprocal_buffer_size;
    unsigned int flags;
    unsigned int num;
    unsigned int gfporder;
    gfp_t gfpflags;
    size_t colour;
    unsigned int colour_off;
    struct kmem_cache *slabp_cache;
    unsigned int slab_size;
    unsigned int dflags;
    void (*ctor)(void *);
    const char *name;
    struct list_head next;
    unsigned long num_active;
    unsigned long num_allocations;
    unsigned long high_mark;
    unsigned long grown;
    unsigned long reaped;
    unsigned long errors;
    unsigned long max_freeable;
    unsigned long node_allocs;
    unsigned long node_frees;
    unsigned long node_overflow;
    atomic_t allochit;
    atomic_t allocmiss;
    atomic_t freehit;
    atomic_t freemiss;
    int obj_offset;
    int obj_size;
    struct kmem_list3 **nodelists;
    struct array_cache *array[2];
}
SIZE: 148

/*怎样从kmem_cache中找到分配的object?

 *应该从struct kmem_list3得到
 **/
crash> rd 0xee29da94 /*得到kmem_list3的地址*/
ee29da94:  ee17fb20                               ...
crash> kmem_list3 ee17fb20
struct kmem_list3 {
  slabs_partial = {
    next = 0xee29e000,
    prev = 0xee29e000
  },
  slabs_full = {/ *kmem_list3的开始地址为0xee17fb20,
         *则成员slabs_full的地址为0xee17fb28，这也说明slabs_full是空的，next/pre都指向它自身；
         */
    next = 0xee17fb28,
    prev = 0xee17fb28
  },
  slabs_free = {
    next = 0xee17fb30,
    prev = 0xee17fb30
  },
  free_objects = 66,
  free_limit = 115,
  colour_next = 0,
  list_lock = {
    {
      rlock = {
        raw_lock = {
          lock = 0
        },
        break_lock = 0
      }
    }
  },
  shared = 0xee27c800,
  alien = 0x0,
  next_reap = 4294953400,
  free_touched = 0
}
crash> kmem_list3
struct kmem_list3 {
    struct list_head slabs_partial;
    struct list_head slabs_full;
    struct list_head slabs_free;
    unsigned long free_objects;
    unsigned int free_limit;
    unsigned int colour_next;
    spinlock_t list_lock;
    struct array_cache *shared;
    struct array_cache **alien;
    unsigned long next_reap;
    int free_touched;
}
SIZE: 60

/*
 crash> kmem_list3 ee17fb20
 struct kmem_list3 {
  slabs_partial = {/*从这里可知分配的 slab的地址为0xee29e000*/
    next = 0xee29e000,
    prev = 0xee29e000
  },
 */
crash> slab
struct slab {
    union {
        struct {
            struct list_head list;
            unsigned long colouroff;
            void *s_mem;
            unsigned int inuse;
            kmem_bufctl_t free;
            unsigned short nodeid;
        };
        struct slab_rcu __slab_cover_slab_rcu;
    };
}
SIZE: 0x1c
crash> struct slab 0xee29e000
struct slab {
  {
    {
      list = {
        next = 0xee17fb20,
        prev = 0xee17fb20
      },
      colouroff = 0x128,
      s_mem = 0xee29e128,
      inuse = 0x1,
      free = 0xe,
      nodeid = 0x0
    },
    __slab_cover_slab_rcu = {
      head = {
        next = 0xee17fb20,
        func = 0xee17fb20
      },
      cachep = 0x128,
      addr = 0xee29e128
    }
  }
}

/*到此，找到了分配的slab,下一步怎样找到object?*/

打印出来的object为：
[    4.085680:1] slab_test: get an object ee29e478
而s_mem = 0xee29e128对不上啊！
明确s_mem指向的是第一个 slab object,且提交到array_cache的个数为batchcount。
分配时从array_cache中读出，且为LIFO，也就是第一个分配的元素为第16个对象。

另为：slab的控制数据保存在哪里？[slab + xxx]
第一个对象为：0xee29e128，slab对象肯定在page的开始部分【page on】
crash> rd ee29e000 128
ee29e000:  ee17fb20 ee17fb20 00000128 ee29e128    ... ...(...(.).
ee29e010:  00000001 0000000e 00000000 00000010   ................
ee29e020:  00000000 00000001 00000002 00000003   ................
ee29e030:  00000004 00000005 00000006 00000007   ................
ee29e040:  00000008 00000009 0000000a 0000000b   ................
ee29e050:  0000000c 0000000d fffffffe 00000011   ................
ee29e060:  00000012 00000013 00000014 00000015   ................
ee29e070:  00000016 00000017 00000018 00000019   ................
ee29e080:  0000001a 0000001b 0000001c 0000001d   ................
ee29e090:  0000001e 0000001f 00000020 00000021   ........ ...!...
ee29e0a0:  00000022 00000023 00000024 00000025   "...#...$...%...
ee29e0b0:  00000026 00000027 00000028 00000029   &...'...(...)...
ee29e0c0:  0000002a 0000002b 0000002c 0000002d   *...+...,...-...
ee29e0d0:  0000002e 0000002f 00000030 00000031   ..../...0...1...
ee29e0e0:  00000032 00000033 00000034 00000035   2...3...4...5...
ee29e0f0:  00000036 00000037 00000038 00000039   6...7...8...9...
ee29e100:  0000003a 0000003b 0000003c 0000003d   :...;...<...=...
ee29e110:  0000003e 0000003f 00000040 00000041   >...?...@...A...
ee29e120:  00000042 ffffffff

/*对象都是从array_cache中分到的*/

{
  array = {0xee1afe60, 0xee1afda0}
}
crash> array_cache
struct array_cache {
    unsigned int avail;
    unsigned int limit;
    unsigned int batchcount;
    unsigned int touched;
    spinlock_t lock;
    void *entry[];
}
SIZE: 24
crash> array_cache 0xee1afe60
struct array_cache {
  avail = 0,
  limit = 32,
  batchcount = 16,
  touched = 0,
  lock = {
    {
      rlock = {
        raw_lock = {
          lock = 0
        },
        break_lock = 0
      }
    }
  },
  entry = 0xee1afe78
}

array_cache的entry的成员什么意思？

array = {0xee1afe60, 0xee1afda0}

crash> array_cache 0xee1afda0
struct array_cache {
  avail = 0,
  limit = 32,
  batchcount = 16,
  touched = 0,
  lock = {
    {
      rlock = {
        raw_lock = {
          lock = 0
        },
        break_lock = 0
      }
    }
  },
  entry = 0xee1afdb8/*这里entry的内容执行 free object*/
}
crash> rd 0xee1afdb8
ee1afdb8:  ee29e438                              8.).

/*这样找来找去，比较慢，可以直接使用命令*/

kmem -S my_cache
crash> kmem -S my_cache
CACHE    NAME                 OBJSIZE  ALLOCATED     TOTAL  SLABS  SSIZE
ee29da00 my_cache                  56          1        67      1     4k
SLAB      MEMORY    TOTAL  ALLOCATED  FREE
ee29e000  ee29e128     67          1    66
FREE / [ALLOCATED]
   ee29e128
   ee29e160
   ee29e198
   ee29e1d0
   ee29e208
   ee29e240
   ee29e278
   ee29e2b0
   ee29e2e8
   ee29e320
   ee29e358
   ee29e390
   ee29e3c8
   ee29e400
   ee29e438
  [ee29e470]/*[]表示分配的，返回的object是ee29e478，为什么不一样？*/
   ee29e4a8
   ee29e4e0
   ee29e518
   ee29e550
   ee29e588
   ee29e5c0
   ee29e5f8
   ee29e630
   ee29e668
   ee29e6a0
   ee29e6d8
   ee29e710
   ee29e748
   ee29e780
   ee29e7b8
   ee29e7f0
   ee29e828
   ee29e860
   ee29e898
   ee29e8d0
   ee29e908
   ee29e940
   ee29e978
   ee29e9b0
   ee29e9e8
   ee29ea20
   ee29ea58
   ee29ea90
   ee29eac8
   ee29eb00
   ee29eb38
   ee29eb70
   ee29eba8
   ee29ebe0
   ee29ec18
   ee29ec50
   ee29ec88
   ee29ecc0
   ee29ecf8
   ee29ed30
   ee29ed68
   ee29eda0
   ee29edd8
   ee29ee10
   ee29ee48
   ee29ee80
   ee29eeb8
   ee29eef0
   ee29ef28
   ee29ef60
   ee29ef98
crash> rd ee29e470 56
ee29e470:  635688c0 d84156c5 5a5a5a5a 5a5a5a5a   ..Vc.VA.ZZZZZZZZ
ee29e480:  5a5a5a5a 5a5a5a5a 5a5a5a5a 5a5a5a5a   ZZZZZZZZZZZZZZZZ
ee29e490:  5a5a5a5a a55a5a5a 635688c0 d84156c5   ZZZZZZZ...Vc.VA.
ee29e4a0:  00000000 c0246140

                 9d74e35b 09f91102   ....@a$.[.t.....
ee29e4b0:  6b6b6b6b 6b6b6b6b 6b6b6b6b 6b6b6b6b   kkkkkkkkkkkkkkkk
ee29e4c0:  6b6b6b6b 6b6b6b6b 6b6b6b6b a56b6b6b   kkkkkkkkkkkkkkk.
ee29e4d0:  9d74e35b 09f91102 00000000 00000000
/**********************************/

从上面可以看出相关的调试信息：

1.SLAB_POISON

  分配的object的内容初始化为5a5a5a5a;
  未分配的object的内容初始化为6b6b6b6b

2.SLAB_RED_ZONE/*从上面可知red zone就是规定好的数值区分为active/inactive,前后都一样*/

  *dbg_redzone1(cachep, objp) = RED_INACTIVE/ACTIVE;
  *dbg_redzone2(cachep, objp) = RED_INACTIVE/ACTIVE;
#define    RED_INACTIVE    0x09F91102 9D74E35BULL    /* when obj is inactive */
#define    RED_ACTIVE    0xD84156C5 635688C0ULL    /* when obj is active */

3. SLAB_STORE_USER

    if (cachep->flags & SLAB_STORE_USER)
        *dbg_userword(cachep, objp) = caller;
    /*从这里可以看出user是最后一个字，上面的例子就是c0246140*/
    static void **dbg_userword(struct kmem_cache *cachep, void *objp)
    {
        BUG_ON(!(cachep->flags & SLAB_STORE_USER));
        return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD);
    }

    crash> dis -l c0246140
    /home/wenshuai/code/kernel3.4/linux_kernel/drivers/misc/tl7689_bp/tl7689_test.c: 58
    0xc0246140 <slab_test+0x40>:    subs    r1, r0, #0