kmalloc

/*************************************************************/
分析 kmalloc的实现过程：

inlude/linux/slab_def.h
static __always_inline void *kmalloc(size_t size, gfp_t flags)
{
    struct kmem_cache *cachep;
    return __kmalloc(size, flags);
}

mm/slab.c:
void *__kmalloc(size_t size, gfp_t flags)
{
    return __do_kmalloc(size, flags, NULL);
}

/**
 * __do_kmalloc - allocate memory
 * @size: how many bytes of memory are required.
 * @flags: the type of memory to allocate (see kmalloc).
 * @caller: function caller for debug tracking of the caller
 */
static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
                      void *caller)
{
    struct kmem_cache *cachep;
    void *ret;

    cachep = __find_general_cachep(size, flags);
    if (unlikely(ZERO_OR_NULL_PTR(cachep)))
        return cachep;
    ret = __cache_alloc(cachep, flags, caller);


    return ret;
}

static inline struct kmem_cache *__find_general_cachep(size_t size,
                            gfp_t gfpflags)
{
    struct cache_sizes *csizep = malloc_sizes;


    if (!size)
        return ZERO_SIZE_PTR;

    while (size > csizep->cs_size)
        csizep++;/*遍历直到找到一个合适的*/

    return csizep->cs_cachep;
}

struct cache_sizes {
    size_t cs_size;
    struct kmem_cache *cs_cachep;
}
SIZE: 8

struct cache_sizes malloc_sizes[] = {
#include <linux/kmalloc_sizes.h>
};

linux/kmalloc_sizes.h
    CACHE(32)
    CACHE(64)
    CACHE(96)
    CACHE(128)
    CACHE(256)
    CACHE(512)
    CACHE(1024)
    CACHE(2048)
    CACHE(4096)
    CACHE(8192)
    CACHE(16384)
    CACHE(32768)
    CACHE(65536)
    CACHE(131072)
    CACHE(262144)

crash> cache_sizes 0xc06e9e1c
struct cache_sizes {
  cs_size = 32,
  cs_cachep = 0xee0000c0
}

crash> kmem_cache 0xee0000c0
struct kmem_cache {
  batchcount = 60,
  limit = 120,
  shared = 8,
  buffer_size = 32,
  reciprocal_buffer_size = 134217728,
  flags = 270336,
  num = 113,
  gfporder = 0,
  gfpflags = 0,
  colour = 0,
  colour_off = 32,
  slabp_cache = 0x0,
  slab_size = 480,
  dflags = 0,
  ctor = 0x0,
  name = 0xc060b16d "size-32",
  next = {
    next = 0xc06e9e00 <cache_cache+64>,
    prev = 0xee000160
  },
  nodelists = 0xee000114,
  array = {0xee00bc00, 0x0, 0xee0025e0, 0x0}
}

static __always_inline void *
__cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
{
    void *objp;
    objp = __do_cache_alloc(cachep, flags);
    return objp;
}

static __always_inline void *
__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
{
    return ____cache_alloc(cachep, flags);
}

static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
{
    void *objp;
    struct array_cache *ac;

    check_irq_off();

     ac = cpu_cache_get(cachep);
    if (likely(ac->avail)) {
        STATS_INC_ALLOCHIT(cachep);
        ac->touched = 1;
        objp = ac->entry[--ac->avail];
    } else {
        STATS_INC_ALLOCMISS(cachep);
         objp = cache_alloc_refill(cachep, flags);
        /*
         * the 'ac' may be updated by cache_alloc_refill(),
         * and kmemleak_erase() requires its correct value.
         */
        ac = cpu_cache_get(cachep);
    }
    return objp;
}

static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
{
    int batchcount;
    struct kmem_list3 *l3;
    struct array_cache *ac;
    int node;

retry:
    check_irq_off();
    node = numa_mem_id();
    ac = cpu_cache_get(cachep);
    batchcount = ac->batchcount;
    if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
        /*
         * If there was little recent activity on this cache, then
         * perform only a partial refill.  Otherwise we could generate
         * refill bouncing.
         */
        batchcount = BATCHREFILL_LIMIT;
    }
    l3 = cachep->nodelists[node];

    BUG_ON(ac->avail > 0 || !l3);
    spin_lock(&l3->list_lock);

    /* See if we can refill from the shared array */
    if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) {
        l3->shared->touched = 1;
        goto alloc_done;
    }

    while (batchcount > 0) {
        struct list_head *entry;
        struct slab *slabp;
        /* Get slab alloc is to come from. */
        entry = l3->slabs_partial.next;
        if (entry == &l3->slabs_partial) {
            l3->free_touched = 1;
            entry = l3->slabs_free.next;
            if (entry == &l3->slabs_free)
                goto must_grow;
        }

        slabp = list_entry(entry, struct slab, list);
        check_slabp(cachep, slabp);
        check_spinlock_acquired(cachep);

        /*
         * The slab was either on partial or free list so
         * there must be at least one object available for
         * allocation.
         */
        BUG_ON(slabp->inuse >= cachep->num);

        while (slabp->inuse < cachep->num && batchcount--) {
            STATS_INC_ALLOCED(cachep);
            STATS_INC_ACTIVE(cachep);
            STATS_SET_HIGH(cachep);

            ac->entry[ac->avail++] = slab_get_obj(cachep, slabp,
                                node);
        }
        check_slabp(cachep, slabp);

        /* move slabp to correct slabp list: */
        list_del(&slabp->list);
        if (slabp->free == BUFCTL_END)
            list_add(&slabp->list, &l3->slabs_full);
        else
            list_add(&slabp->list, &l3->slabs_partial);
    }

must_grow:
    l3->free_objects -= ac->avail;
alloc_done:
    spin_unlock(&l3->list_lock);

    if (unlikely(!ac->avail)) {
        int x;
        x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL);

        /* cache_grow can reenable interrupts, then ac could change. */
        ac = cpu_cache_get(cachep);
        if (!x && ac->avail == 0)    /* no objects in sight? abort */
            return NULL;

        if (!ac->avail)        /* objects refilled by interrupt? */
            goto retry;
    }
    ac->touched = 1;
    return ac->entry[--ac->avail];
}


/*
 * Grow (by 1) the number of slabs within a cache.  This is called by
 * kmem_cache_alloc() when there are no active objs left in a cache.
 */
static int cache_grow(struct kmem_cache *cachep,
        gfp_t flags, int nodeid, void *objp)
{
    struct slab *slabp;
    size_t offset;
    gfp_t local_flags;
    struct kmem_list3 *l3;

    /*
     * Be lazy and only check for valid flags here,  keeping it out of the
     * critical path in kmem_cache_alloc().
     */
    BUG_ON(flags & GFP_SLAB_BUG_MASK);
    local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);

    /* Take the l3 list lock to change the colour_next on this node */
    check_irq_off();
    l3 = cachep->nodelists[nodeid];
    spin_lock(&l3->list_lock);

    /* Get colour for the slab, and cal the next value. */
    offset = l3->colour_next;
    l3->colour_next++;
    if (l3->colour_next >= cachep->colour)
        l3->colour_next = 0;
    spin_unlock(&l3->list_lock);

    offset *= cachep->colour_off;

    if (local_flags & __GFP_WAIT)
        local_irq_enable();

    /*
     * The test for missing atomic flag is performed here, rather than
     * the more obvious place, simply to reduce the critical path length
     * in kmem_cache_alloc(). If a caller is seriously mis-behaving they
     * will eventually be caught here (where it matters).
     */
    kmem_flagcheck(cachep, flags);

    /*
     * Get mem for the objs.  Attempt to allocate a physical page from
     * 'nodeid'.
     */
    if (!objp)
        objp = kmem_getpages(cachep, local_flags, nodeid);
    if (!objp)
        goto failed;

    /* Get slab management. */
    slabp = alloc_slabmgmt(cachep, objp, offset,
            local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
    if (!slabp)
        goto opps1;

    slab_map_pages(cachep, slabp, objp);

    cache_init_objs(cachep, slabp);

    if (local_flags & __GFP_WAIT)
        local_irq_disable();
    check_irq_off();
    spin_lock(&l3->list_lock);

    /* Make slab active. */
    list_add_tail(&slabp->list, &(l3->slabs_free));
    STATS_INC_GROWN(cachep);
    l3->free_objects += cachep->num;
    spin_unlock(&l3->list_lock);
    return 1;
opps1:
    kmem_freepages(cachep, objp);
failed:
    if (local_flags & __GFP_WAIT)
        local_irq_disable();
    return 0;
}

struct array_cache {
    unsigned int avail;
    unsigned int limit;
    unsigned int batchcount;
    unsigned int touched;
    spinlock_t lock;
    void *entry[];    /*
             * Must have this definition in here for the proper
             * alignment of array_cache. Also simplifies accessing
             * the entries.
             */
};
void __init kmem_cache_init(void)

mm_init -> kmem_cache_init();


crash> kmem_cache
struct kmem_cache {
    unsigned int batchcount;
    unsigned int limit;
    unsigned int shared;
    unsigned int buffer_size;
    u32 reciprocal_buffer_size;
    unsigned int flags;
    unsigned int num;
    unsigned int gfporder;
    gfp_t gfpflags;
    size_t colour;
    unsigned int colour_off;
    struct kmem_cache *slabp_cache;
    unsigned int slab_size;
    unsigned int dflags;
    void (*ctor)(void *);
    const char *name;
    struct list_head next;
    struct kmem_list3 **nodelists;
    struct array_cache *array[4];
}
SIZE: 92

crash> kmem_list3
struct kmem_list3 {
    struct list_head slabs_partial;
    struct list_head slabs_full;
    struct list_head slabs_free;
    unsigned long free_objects;
    unsigned int free_limit;
    unsigned int colour_next;
    spinlock_t list_lock;
    struct array_cache *shared;
    struct array_cache **alien;
    unsigned long next_reap;
    int free_touched;
}
SIZE: 60

crash> slab
struct slab {
    union {
        struct {
            struct list_head list;
            unsigned long colouroff;
            void *s_mem;
            unsigned int inuse;
            kmem_bufctl_t free;
            unsigned short nodeid;
        };
        struct slab_rcu __slab_cover_slab_rcu;
    };
}
SIZE: 28