kmalloc

/**
 * kmalloc - allocate memory
 * @size: how many bytes of memory are required.
 * @flags: the type of memory to allocate.
 *
 * kmalloc is the normal method of allocating memory
 * for objects smaller than page size in the kernel.
 *
 * The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN
 * bytes. For @size of power of two bytes, the alignment is also guaranteed
 * to be at least to the size.
 *
 * The @flags argument may be one of the GFP flags defined at
 * include/linux/gfp.h and described at
 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>`
 *
 * The recommended usage of the @flags is described at
 * :ref:`Documentation/core-api/memory-allocation.rst <memory-allocation>`
 *
 * Below is a brief outline of the most useful GFP flags
 *
 * %GFP_KERNEL
 *    Allocate normal kernel ram. May sleep.
 *
 * %GFP_NOWAIT
 *    Allocation will not sleep.
 *
 * %GFP_ATOMIC
 *    Allocation will not sleep.  May use emergency pools.
 *
 * %GFP_HIGHUSER
 *    Allocate memory from high memory on behalf of user.
 *
 * Also it is possible to set different flags by OR'ing
 * in one or more of the following additional @flags:
 *
 * %__GFP_HIGH
 *    This allocation has high priority and may use emergency pools.
 *
 * %__GFP_NOFAIL
 *    Indicate that this allocation is in no way allowed to fail
 *    (think twice before using).
 *
 * %__GFP_NORETRY
 *    If memory is not immediately available,
 *    then give up at once.
 *
 * %__GFP_NOWARN
 *    If allocation fails, don't issue any warnings.
 *
 * %__GFP_RETRY_MAYFAIL
 *    Try really hard to succeed the allocation but fail
 *    eventually.
 */
static __always_inline void *kmalloc(size_t size, gfp_t flags)
{
    if (__builtin_constant_p(size)) {
#ifndef CONFIG_SLOB
        unsigned int index;
#endif
        if (size > KMALLOC_MAX_CACHE_SIZE)
            return kmalloc_large(size, flags);
#ifndef CONFIG_SLOB
        index = kmalloc_index(size);

        if (!index)
            return ZERO_SIZE_PTR;

        return kmem_cache_alloc_trace(
                kmalloc_caches[kmalloc_type(flags)][index],
                flags, size);
#endif
    }
    return __kmalloc(size, flags);
}

static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
{
    unsigned int order = get_order(size);
    return kmalloc_order_trace(size, flags, order);
}

/**
 * get_order - Determine the allocation order of a memory size
 * @size: The size for which to get the order
 *
 * Determine the allocation order of a particular sized block of memory.  This
 * is on a logarithmic scale, where:
 *
 *    0 -> 2^0 * PAGE_SIZE and below
 *    1 -> 2^1 * PAGE_SIZE to 2^0 * PAGE_SIZE + 1
 *    2 -> 2^2 * PAGE_SIZE to 2^1 * PAGE_SIZE + 1
 *    3 -> 2^3 * PAGE_SIZE to 2^2 * PAGE_SIZE + 1
 *    4 -> 2^4 * PAGE_SIZE to 2^3 * PAGE_SIZE + 1
 *    ...
 *
 * The order returned is used to find the smallest allocation granule required
 * to hold an object of the specified size.
 *
 * The result is undefined if the size is 0.
 */
static inline __attribute_const__ int get_order(unsigned long size)
{
    if (__builtin_constant_p(size)) {
        if (!size)
            return BITS_PER_LONG - PAGE_SHIFT;

        if (size < (1UL << PAGE_SHIFT))
            return 0;

        return ilog2((size) - 1) - PAGE_SHIFT + 1;
    }

    size--;
    size >>= PAGE_SHIFT;
#if BITS_PER_LONG == 32
    return fls(size);
#else
    return fls64(size);
#endif
}

void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
{
    void *ret = kmalloc_order(size, flags, order);
    trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
    return ret;
}

/*
 * To avoid unnecessary overhead, we pass through large allocation requests
 * directly to the page allocator. We use __GFP_COMP, because we will need to
 * know the allocation order to free the pages properly in kfree.
 */
void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
{
    void *ret = NULL;
    struct page *page;

    flags |= __GFP_COMP;
    page = alloc_pages(flags, order);
    if (likely(page)) {
        ret = page_address(page);
        mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE,
                    1 << order);
    }
    ret = kasan_kmalloc_large(ret, size, flags);
    /* As ret might get tagged, call kmemleak hook after KASAN. */
    kmemleak_alloc(ret, size, 1, flags);
    return ret;
}