FreeBSD zone allocator

　　FreeBSD zone allocator 实现为一个 slab allocator ，但在细节方面和原始的 slab allocator 不太一样。

 

　　在 zone allocator 中，keg 充当着 back end 的角色，zone 起着 front end 的作用。

 　　主要的数据结构有：struct uma_keg , struct uma_zone , struct uma_slab , struct uma_cache , struct uma_bucket。

 

/* * Keg management structure * * TODO: Optimize for cache line size * */ struct uma_keg { LIST_ENTRY(uma_keg) uk_link; /* List of all kegs */ struct mtx uk_lock; /* Lock for the keg */ struct uma_hash uk_hash; LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */ LIST_HEAD(,uma_slab) uk_part_slab; /* partially allocated slabs */ LIST_HEAD(,uma_slab) uk_free_slab; /* empty slab list */ LIST_HEAD(,uma_slab) uk_full_slab; /* full slabs */ u_int32_t uk_recurse; /* Allocation recursion count */ u_int32_t uk_align; /* Alignment mask */ u_int32_t uk_pages; /* Total page count */ u_int32_t uk_free; /* Count of items free in slabs */ u_int32_t uk_size; /* Requested size of each item */ u_int32_t uk_rsize; /* Real size of each item */ u_int32_t uk_maxpages; /* Maximum number of pages to alloc */ uma_init uk_init; /* Keg's init routine */ uma_fini uk_fini; /* Keg's fini routine */ uma_alloc uk_allocf; /* Allocation function */ uma_free uk_freef; /* Free routine */ struct vm_object *uk_obj; /* Zone specific object */ vm_offset_t uk_kva; /* Base kva for zones with objs */ uma_zone_t uk_slabzone; /* Slab zone backing us, if OFFPAGE */ u_int16_t uk_pgoff; /* Offset to uma_slab struct */ u_int16_t uk_ppera; /* pages per allocation from backend */ u_int16_t uk_ipers; /* Items per slab */ u_int32_t uk_flags; /* Internal flags */ };

 

　　keg 存储着三类 slab 的链表：uk_free_slab（完全没有使用的 slab ），uk_part_slab（部分使用的 slab），uk_full_slab（全部使用了的 slab）。uk_link链表连接这系统中所有的 keg ，链表头存储在 uma_kegs中，以便 zone_foreach 遍历所有的 keg。

/* Page management structure */ /* Sorry for the union, but space efficiency is important */ struct uma_slab_head { uma_keg_t us_keg; /* Keg we live in */ union { LIST_ENTRY(uma_slab) _us_link; /* slabs in zone */ unsigned long _us_size; /* Size of allocation */ } us_type; SLIST_ENTRY(uma_slab) us_hlink; /* Link for hash table */ u_int8_t *us_data; /* First item */ u_int8_t us_flags; /* Page flags see uma.h */ u_int8_t us_freecount; /* How many are free? */ u_int8_t us_firstfree; /* First free item index */ }; /* The standard slab structure */ struct uma_slab { struct uma_slab_head us_head; /* slab header data */ struct { u_int8_t us_item; } us_freelist[1]; /* actual number bigger */ }; /* * The slab structure for UMA_ZONE_REFCNT zones for whose items we * maintain reference counters in the slab for. */ struct uma_slab_refcnt { struct uma_slab_head us_head; /* slab header data */ struct { u_int8_t us_item; u_int32_t us_refcnt; } us_freelist[1]; /* actual number bigger */ }; #define us_keg us_head.us_keg #define us_link us_head.us_type._us_link #define us_size us_head.us_type._us_size #define us_hlink us_head.us_hlink #define us_data us_head.us_data #define us_flags us_head.us_flags #define us_freecount us_head.us_freecount #define us_firstfree us_head.us_firstfree

　　us_link 连接着所有属于同类的 slab ，当用 uma_large_malloc 分配时，slab 不被 keg 管理，用 us_size 保存数据的大小。

/* * Structures for per cpu queues. */ struct uma_bucket { LIST_ENTRY(uma_bucket) ub_link; /* Link into the zone */ int16_t ub_cnt; /* Count of free items. */ int16_t ub_entries; /* Max items. */ void *ub_bucket[]; /* actual allocation storage */ }; typedef struct uma_bucket * uma_bucket_t; struct uma_cache { uma_bucket_t uc_freebucket; /* Bucket we're freeing to */ uma_bucket_t uc_allocbucket; /* Bucket to allocate from */ u_int64_t uc_allocs; /* Count of allocations */ u_int64_t uc_frees; /* Count of frees */ };

　　当分配内存时，首先检查当前 CPU 的 uc_freebucket 和 uc_allocbucket 是否有缓存的 item；当释放内存时，首先检查当前 CPU 的 uc_freebucket 和 uc_allocbucket 是否有空槽可以缓存释放的 item。uc_allocs 和 uc_frees 保存着迄今为止只在 cache 内进行的分配和释放数。

 

/* * Zone management structure * * TODO: Optimize for cache line size * */ struct uma_zone { char *uz_name; /* Text name of the zone */ struct mtx *uz_lock; /* Lock for the zone (keg's lock) */ uma_keg_t uz_keg; /* Our underlying Keg */ LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ LIST_HEAD(,uma_bucket) uz_full_bucket; /* full buckets */ LIST_HEAD(,uma_bucket) uz_free_bucket; /* Buckets for frees */ uma_ctor uz_ctor; /* Constructor for each allocation */ uma_dtor uz_dtor; /* Destructor */ uma_init uz_init; /* Initializer for each item */ uma_fini uz_fini; /* Discards memory */ u_int64_t uz_allocs; /* Total number of allocations */ u_int64_t uz_frees; /* Total number of frees */ u_int64_t uz_fails; /* Total number of alloc failures */ uint16_t uz_fills; /* Outstanding bucket fills */ uint16_t uz_count; /* Highest value ub_ptr can have */ /* * This HAS to be the last item because we adjust the zone size * based on NCPU and then allocate the space for the zones. */ struct uma_cache uz_cpu[1]; /* Per cpu caches */ };

　　uz_full_bucket 链表连接着所有缓存着 item 的 bucket ，uz_free_bucket 链表连接着所有没有缓存着 item 的 bucket。分配内存时，如 CPU 的 cache 内没有缓存的 item 时，可将 uz_full_bucket 链表中的 bucket 移到 CPU 的 cache 内以分配内存；释放内存时，如 CPU 的 cache 内没有空槽可以缓存释放的 item ，可将 uz_free_bucket 链表中的bucket 移到 CPU 的 cache 内以缓存内存。

　　对 keg 进行 grow 操作的主要函数有：uma_zone_slab，slab_zalloc。

 

static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags) { uma_slab_t slab; uma_keg_t keg; keg = zone->uz_keg; /* * This is to prevent us from recursively trying to allocate * buckets. The problem is that if an allocation forces us to * grab a new bucket we will call page_alloc, which will go off * and cause the vm to allocate vm_map_entries. If we need new * buckets there too we will recurse in kmem_alloc and bad * things happen. So instead we return a NULL bucket, and make * the code that allocates buckets smart enough to deal with it * * XXX: While we want this protection for the bucket zones so that * recursion from the VM is handled (and the calling code that * allocates buckets knows how to deal with it), we do not want * to prevent allocation from the slab header zones (slabzone * and slabrefzone) if uk_recurse is not zero for them. The * reason is that it could lead to NULL being returned for * slab header allocations even in the M_WAITOK case, and the * caller can't handle that. */ if (keg->uk_flags & UMA_ZFLAG_INTERNAL && keg->uk_recurse != 0) if (zone != slabzone && zone != slabrefzone && zone != zones) return (NULL); slab = NULL; for (;;) { /* * Find a slab with some space. Prefer slabs that are partially * used over those that are totally full. This helps to reduce * fragmentation. */ if (keg->uk_free != 0) { if (!LIST_EMPTY(&keg->uk_part_slab)) { slab = LIST_FIRST(&keg->uk_part_slab); } else { slab = LIST_FIRST(&keg->uk_free_slab); LIST_REMOVE(slab, us_link); LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); } return (slab); } /* * M_NOVM means don't ask at all! */ if (flags & M_NOVM) break; if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) { keg->uk_flags |= UMA_ZFLAG_FULL; if (flags & M_NOWAIT) break; else msleep(keg, &keg->uk_lock, PVM, "zonelimit", 0); continue; } keg->uk_recurse++; slab = slab_zalloc(zone, flags); keg->uk_recurse--; /* * If we got a slab here it's safe to mark it partially used * and return. We assume that the caller is going to remove * at least one item. */ if (slab) { LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); return (slab); } /* * We might not have been able to get a slab but another cpu * could have while we were unlocked. Check again before we * fail. */ if (flags & M_NOWAIT) flags |= M_NOVM; } return (slab); }

 

 

/* * Allocate a new slab for a zone. This does not insert the slab onto a list. * * Arguments: * zone The zone to allocate slabs for * wait Shall we wait? * * Returns: * The slab that was allocated or NULL if there is no memory and the * caller specified M_NOWAIT. */ static uma_slab_t slab_zalloc(uma_zone_t zone, int wait) { uma_slabrefcnt_t slabref; uma_slab_t slab; uma_keg_t keg; u_int8_t *mem; u_int8_t flags; int i; slab = NULL; keg = zone->uz_keg; #ifdef UMA_DEBUG printf("slab_zalloc: Allocating a new slab for %s/n", zone->uz_name); #endif ZONE_UNLOCK(zone); if (keg->uk_flags & UMA_ZONE_OFFPAGE) { slab = uma_zalloc_internal(keg->uk_slabzone, NULL, wait); if (slab == NULL) { ZONE_LOCK(zone); return NULL; } } /* * This reproduces the old vm_zone behavior of zero filling pages the * first time they are added to a zone. * * Malloced items are zeroed in uma_zalloc. */ if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) wait |= M_ZERO; else wait &= ~M_ZERO; mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE, &flags, wait); if (mem == NULL) { if (keg->uk_flags & UMA_ZONE_OFFPAGE) uma_zfree_internal(keg->uk_slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE); ZONE_LOCK(zone); return (NULL); } /* Point the slab into the allocated memory */ if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) slab = (uma_slab_t )(mem + keg->uk_pgoff); if ((keg->uk_flags & UMA_ZONE_MALLOC) || (keg->uk_flags & UMA_ZONE_REFCNT)) for (i = 0; i < keg->uk_ppera; i++) vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); slab->us_keg = keg; slab->us_data = mem; slab->us_freecount = keg->uk_ipers; slab->us_firstfree = 0; slab->us_flags = flags; if (keg->uk_flags & UMA_ZONE_REFCNT) { slabref = (uma_slabrefcnt_t)slab; for (i = 0; i < keg->uk_ipers; i++) { slabref->us_freelist[i].us_refcnt = 0; slabref->us_freelist[i].us_item = i+1; } } else { for (i = 0; i < keg->uk_ipers; i++) slab->us_freelist[i].us_item = i+1; } if (keg->uk_init != NULL) { for (i = 0; i < keg->uk_ipers; i++) if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), keg->uk_size, wait) != 0) break; if (i != keg->uk_ipers) { if (keg->uk_fini != NULL) { for (i--; i > -1; i--) keg->uk_fini(slab->us_data + (keg->uk_rsize * i), keg->uk_size); } if ((keg->uk_flags & UMA_ZONE_MALLOC) || (keg->uk_flags & UMA_ZONE_REFCNT)) { vm_object_t obj; if (flags & UMA_SLAB_KMEM) obj = kmem_object; else if (flags & UMA_SLAB_KERNEL) obj = kernel_object; else obj = NULL; for (i = 0; i < keg->uk_ppera; i++) vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), obj); } if (keg->uk_flags & UMA_ZONE_OFFPAGE) uma_zfree_internal(keg->uk_slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE); keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags); ZONE_LOCK(zone); return (NULL); } } ZONE_LOCK(zone); if (keg->uk_flags & UMA_ZONE_HASH) UMA_HASH_INSERT(&keg->uk_hash, slab, mem); keg->uk_pages += keg->uk_ppera; keg->uk_free += keg->uk_ipers; return (slab); }

 

　　uma_zone_slab 函数返回没有全部使用了的 slab 。如果调用 uma_zone_slab 时，keg 的 uk_free 为0（此时，uk_free_slab 和 uk_part_slab 链表均为空），则调用 slab_zalloc 分配新的 slab 。slab_zalloc 通过调用 keg->uk_allocf来分配 item 块内存，根据 keg->uk_flag 的标识来决定 slab 是从 keg->uk_slabzone 中分配还是存储在 item 块的最后面。如果，slab_zalloc 分配 item 块成功，keg->uk_pages 将增加 keg->uk_ppera 。当对相同的 keg 调用 uma_zone_slab 时，如果 uk_free 为 0，当 uk_maxpages 设置并且 uk_pages 大于等于 uk_maxpages 时，函数将阻塞在 uk_lock 上（见 uma_zone_slab 1977-2005行）。

　　uma_zalloc_arg 函数从 zone 分配 item 。

void * uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) { void *item; uma_cache_t cache; uma_bucket_t bucket; int cpu; /* This is the fast path allocation */ #ifdef UMA_DEBUG_ALLOC_1 printf("Allocating one item from %s(%p)/n", zone->uz_name, zone); #endif CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread, zone->uz_name, flags); if (flags & M_WAITOK) { WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "uma_zalloc_arg: zone /"%s/"", zone->uz_name); } /* * If possible, allocate from the per-CPU cache. There are two * requirements for safe access to the per-CPU cache: (1) the thread * accessing the cache must not be preempted or yield during access, * and (2) the thread must not migrate CPUs without switching which * cache it accesses. We rely on a critical section to prevent * preemption and migration. We release the critical section in * order to acquire the zone mutex if we are unable to allocate from * the current cache; when we re-acquire the critical section, we * must detect and handle migration if it has occurred. */ zalloc_restart: critical_enter(); cpu = curcpu; cache = &zone->uz_cpu[cpu]; zalloc_start: bucket = cache->uc_allocbucket; if (bucket) { if (bucket->ub_cnt > 0) { bucket->ub_cnt--; item = bucket->ub_bucket[bucket->ub_cnt]; #ifdef INVARIANTS bucket->ub_bucket[bucket->ub_cnt] = NULL; #endif KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled.")); cache->uc_allocs++; critical_exit(); #ifdef INVARIANTS ZONE_LOCK(zone); uma_dbg_alloc(zone, NULL, item); ZONE_UNLOCK(zone); #endif if (zone->uz_ctor != NULL) { if (zone->uz_ctor(item, zone->uz_keg->uk_size, udata, flags) != 0) { uma_zfree_internal(zone, item, udata, SKIP_DTOR, ZFREE_STATFAIL | ZFREE_STATFREE); return (NULL); } } if (flags & M_ZERO) bzero(item, zone->uz_keg->uk_size); return (item); } else if (cache->uc_freebucket) { /* * We have run out of items in our allocbucket. * See if we can switch with our free bucket. */ if (cache->uc_freebucket->ub_cnt > 0) { #ifdef UMA_DEBUG_ALLOC printf("uma_zalloc: Swapping empty with" " alloc./n"); #endif bucket = cache->uc_freebucket; cache->uc_freebucket = cache->uc_allocbucket; cache->uc_allocbucket = bucket; goto zalloc_start; } } } /* * Attempt to retrieve the item from the per-CPU cache has failed, so * we must go back to the zone. This requires the zone lock, so we * must drop the critical section, then re-acquire it when we go back * to the cache. Since the critical section is released, we may be * preempted or migrate. As such, make sure not to maintain any * thread-local state specific to the cache from prior to releasing * the critical section. */ critical_exit(); ZONE_LOCK(zone); critical_enter(); cpu = curcpu; cache = &zone->uz_cpu[cpu]; bucket = cache->uc_allocbucket; if (bucket != NULL) { if (bucket->ub_cnt > 0) { ZONE_UNLOCK(zone); goto zalloc_start; } bucket = cache->uc_freebucket; if (bucket != NULL && bucket->ub_cnt > 0) { ZONE_UNLOCK(zone); goto zalloc_start; } } /* Since we have locked the zone we may as well send back our stats */ zone->uz_allocs += cache->uc_allocs; cache->uc_allocs = 0; zone->uz_frees += cache->uc_frees; cache->uc_frees = 0; /* Our old one is now a free bucket */ if (cache->uc_allocbucket) { KASSERT(cache->uc_allocbucket->ub_cnt == 0, ("uma_zalloc_arg: Freeing a non free bucket.")); LIST_INSERT_HEAD(&zone->uz_free_bucket, cache->uc_allocbucket, ub_link); cache->uc_allocbucket = NULL; } /* Check the free list for a new alloc bucket */ if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { KASSERT(bucket->ub_cnt != 0, ("uma_zalloc_arg: Returning an empty bucket.")); LIST_REMOVE(bucket, ub_link); cache->uc_allocbucket = bucket; ZONE_UNLOCK(zone); goto zalloc_start; } /* We are no longer associated with this CPU. */ critical_exit(); /* Bump up our uz_count so we get here less */ if (zone->uz_count < BUCKET_MAX) zone->uz_count++; /* * Now lets just fill a bucket and put it on the free list. If that * works we'll restart the allocation from the begining. */ if (uma_zalloc_bucket(zone, flags)) { ZONE_UNLOCK(zone); goto zalloc_restart; } ZONE_UNLOCK(zone); /* * We may not be able to get a bucket so return an actual item. */ #ifdef UMA_DEBUG printf("uma_zalloc_arg: Bucketzone returned NULL/n"); #endif return (uma_zalloc_internal(zone, udata, flags)); }

　　uma_zalloc_arg 首先尝试从当前 CPU cache 中分配 item 。如失败：若 zone 的 uz_full_bucket 链表不为空，则将其首元素移到 CPU cache 中以供分配；否则调用 uma_zalloc_bucket 装填一个新的 bucket ，并将之插入 uz_full_bucket 链表。如果 uma_zalloc_bucket 也失败，则调用 uma_zalloc_internal 分配内存。

static int uma_zalloc_bucket(uma_zone_t zone, int flags) { uma_bucket_t bucket; uma_slab_t slab; int16_t saved; int max, origflags = flags; /* * Try this zone's free list first so we don't allocate extra buckets. */ if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { KASSERT(bucket->ub_cnt == 0, ("uma_zalloc_bucket: Bucket on free list is not empty.")); LIST_REMOVE(bucket, ub_link); } else { int bflags; bflags = (flags & ~M_ZERO); if (zone->uz_keg->uk_flags & UMA_ZFLAG_CACHEONLY) bflags |= M_NOVM; ZONE_UNLOCK(zone); bucket = bucket_alloc(zone->uz_count, bflags); ZONE_LOCK(zone); } if (bucket == NULL) return (0); #ifdef SMP /* * This code is here to limit the number of simultaneous bucket fills * for any given zone to the number of per cpu caches in this zone. This * is done so that we don't allocate more memory than we really need. */ if (zone->uz_fills >= mp_ncpus) goto done; #endif zone->uz_fills++; max = MIN(bucket->ub_entries, zone->uz_count); /* Try to keep the buckets totally full */ saved = bucket->ub_cnt; while (bucket->ub_cnt < max && (slab = uma_zone_slab(zone, flags)) != NULL) { while (slab->us_freecount && bucket->ub_cnt < max) { bucket->ub_bucket[bucket->ub_cnt++] = uma_slab_alloc(zone, slab); } /* Don't block on the next fill */ flags |= M_NOWAIT; } /* * We unlock here because we need to call the zone's init. * It should be safe to unlock because the slab dealt with * above is already on the appropriate list within the keg * and the bucket we filled is not yet on any list, so we * own it. */ if (zone->uz_init != NULL) { int i; ZONE_UNLOCK(zone); for (i = saved; i < bucket->ub_cnt; i++) if (zone->uz_init(bucket->ub_bucket[i], zone->uz_keg->uk_size, origflags) != 0) break; /* * If we couldn't initialize the whole bucket, put the * rest back onto the freelist. */ if (i != bucket->ub_cnt) { int j; for (j = i; j < bucket->ub_cnt; j++) { uma_zfree_internal(zone, bucket->ub_bucket[j], NULL, SKIP_FINI, 0); #ifdef INVARIANTS bucket->ub_bucket[j] = NULL; #endif } bucket->ub_cnt = i; } ZONE_LOCK(zone); } zone->uz_fills--; if (bucket->ub_cnt != 0) { LIST_INSERT_HEAD(&zone->uz_full_bucket, bucket, ub_link); return (1); } #ifdef SMP done: #endif bucket_free(bucket); return (0); } /* * Allocates an item for an internal zone * * Arguments * zone The zone to alloc for. * udata The data to be passed to the constructor. * flags M_WAITOK, M_NOWAIT, M_ZERO. * * Returns * NULL if there is no memory and M_NOWAIT is set * An item if successful */ static void * uma_zalloc_internal(uma_zone_t zone, void *udata, int flags) { uma_keg_t keg; uma_slab_t slab; void *item; item = NULL; keg = zone->uz_keg; #ifdef UMA_DEBUG_ALLOC printf("INTERNAL: Allocating one item from %s(%p)/n", zone->uz_name, zone); #endif ZONE_LOCK(zone); slab = uma_zone_slab(zone, flags); if (slab == NULL) { zone->uz_fails++; ZONE_UNLOCK(zone); return (NULL); } item = uma_slab_alloc(zone, slab); zone->uz_allocs++; ZONE_UNLOCK(zone); /* * We have to call both the zone's init (not the keg's init) * and the zone's ctor. This is because the item is going from * a keg slab directly to the user, and the user is expecting it * to be both zone-init'd as well as zone-ctor'd. */ if (zone->uz_init != NULL) { if (zone->uz_init(item, keg->uk_size, flags) != 0) { uma_zfree_internal(zone, item, udata, SKIP_FINI, ZFREE_STATFAIL | ZFREE_STATFREE); return (NULL); } } if (zone->uz_ctor != NULL) { if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) { uma_zfree_internal(zone, item, udata, SKIP_DTOR, ZFREE_STATFAIL | ZFREE_STATFREE); return (NULL); } } if (flags & M_ZERO) bzero(item, keg->uk_size); return (item); }

　　uma_zalloc_internal 函数先调用 uma_zone_slab 函数返回一个未用完的 slab ，再调用 uma_slab_alloc 函数从这个 slab 中分配一个 item 。uma_zone_slab 函数假设调用程序至少从返回的 slab 里分配一个 item ，所以如果返回的 slab 是完全没有使用的，则在返回之前将之从 uk_free_slab 中移除，并插入 uk_part_slab 。uma_slab_alloc 函数假设 slab 是部分使用了的 slab ，所以只处理当 slab->us_freecount 等于 0 的情况。

　　

　　uma_zfree_arg 将从 zone 分配的 item 返回给 zone 。

void uma_zfree_arg(uma_zone_t zone, void *item, void *udata) { uma_keg_t keg; uma_cache_t cache; uma_bucket_t bucket; int bflags; int cpu; keg = zone->uz_keg; #ifdef UMA_DEBUG_ALLOC_1 printf("Freeing item %p to %s(%p)/n", item, zone->uz_name, zone); #endif CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, zone->uz_name); if (zone->uz_dtor) zone->uz_dtor(item, keg->uk_size, udata); #ifdef INVARIANTS ZONE_LOCK(zone); if (keg->uk_flags & UMA_ZONE_MALLOC) uma_dbg_free(zone, udata, item); else uma_dbg_free(zone, NULL, item); ZONE_UNLOCK(zone); #endif /* * The race here is acceptable. If we miss it we'll just have to wait * a little longer for the limits to be reset. */ if (keg->uk_flags & UMA_ZFLAG_FULL) goto zfree_internal; /* * If possible, free to the per-CPU cache. There are two * requirements for safe access to the per-CPU cache: (1) the thread * accessing the cache must not be preempted or yield during access, * and (2) the thread must not migrate CPUs without switching which * cache it accesses. We rely on a critical section to prevent * preemption and migration. We release the critical section in * order to acquire the zone mutex if we are unable to free to the * current cache; when we re-acquire the critical section, we must * detect and handle migration if it has occurred. */ zfree_restart: critical_enter(); cpu = curcpu; cache = &zone->uz_cpu[cpu]; zfree_start: bucket = cache->uc_freebucket; if (bucket) { /* * Do we have room in our bucket? It is OK for this uz count * check to be slightly out of sync. */ if (bucket->ub_cnt < bucket->ub_entries) { KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, ("uma_zfree: Freeing to non free bucket index.")); bucket->ub_bucket[bucket->ub_cnt] = item; bucket->ub_cnt++; cache->uc_frees++; critical_exit(); return; } else if (cache->uc_allocbucket) { #ifdef UMA_DEBUG_ALLOC printf("uma_zfree: Swapping buckets./n"); #endif /* * We have run out of space in our freebucket. * See if we can switch with our alloc bucket. */ if (cache->uc_allocbucket->ub_cnt < cache->uc_freebucket->ub_cnt) { bucket = cache->uc_freebucket; cache->uc_freebucket = cache->uc_allocbucket; cache->uc_allocbucket = bucket; goto zfree_start; } } } /* * We can get here for two reasons: * * 1) The buckets are NULL * 2) The alloc and free buckets are both somewhat full. * * We must go back the zone, which requires acquiring the zone lock, * which in turn means we must release and re-acquire the critical * section. Since the critical section is released, we may be * preempted or migrate. As such, make sure not to maintain any * thread-local state specific to the cache from prior to releasing * the critical section. */ critical_exit(); ZONE_LOCK(zone); critical_enter(); cpu = curcpu; cache = &zone->uz_cpu[cpu]; if (cache->uc_freebucket != NULL) { if (cache->uc_freebucket->ub_cnt < cache->uc_freebucket->ub_entries) { ZONE_UNLOCK(zone); goto zfree_start; } if (cache->uc_allocbucket != NULL && (cache->uc_allocbucket->ub_cnt < cache->uc_freebucket->ub_cnt)) { ZONE_UNLOCK(zone); goto zfree_start; } } /* Since we have locked the zone we may as well send back our stats */ zone->uz_allocs += cache->uc_allocs; cache->uc_allocs = 0; zone->uz_frees += cache->uc_frees; cache->uc_frees = 0; bucket = cache->uc_freebucket; cache->uc_freebucket = NULL; /* Can we throw this on the zone full list? */ if (bucket != NULL) { #ifdef UMA_DEBUG_ALLOC printf("uma_zfree: Putting old bucket on the free list./n"); #endif /* ub_cnt is pointing to the last free item */ KASSERT(bucket->ub_cnt != 0, ("uma_zfree: Attempting to insert an empty bucket onto the full list./n")); LIST_INSERT_HEAD(&zone->uz_full_bucket, bucket, ub_link); } if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { LIST_REMOVE(bucket, ub_link); ZONE_UNLOCK(zone); cache->uc_freebucket = bucket; goto zfree_start; } /* We are no longer associated with this CPU. */ critical_exit(); /* And the zone.. */ ZONE_UNLOCK(zone); #ifdef UMA_DEBUG_ALLOC printf("uma_zfree: Allocating new free bucket./n"); #endif bflags = M_NOWAIT; if (keg->uk_flags & UMA_ZFLAG_CACHEONLY) bflags |= M_NOVM; bucket = bucket_alloc(zone->uz_count, bflags); if (bucket) { ZONE_LOCK(zone); LIST_INSERT_HEAD(&zone->uz_free_bucket, bucket, ub_link); ZONE_UNLOCK(zone); goto zfree_restart; } /* * If nothing else caught this, we'll just do an internal free. */ zfree_internal: uma_zfree_internal(zone, item, udata, SKIP_DTOR, ZFREE_STATFREE); return; }

/* * Frees an item to an INTERNAL zone or allocates a free bucket * * Arguments: * zone The zone to free to * item The item we're freeing * udata User supplied data for the dtor * skip Skip dtors and finis */ static void uma_zfree_internal(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip, int flags) { uma_slab_t slab; uma_slabrefcnt_t slabref; uma_keg_t keg; u_int8_t *mem; u_int8_t freei; keg = zone->uz_keg; if (skip < SKIP_DTOR && zone->uz_dtor) zone->uz_dtor(item, keg->uk_size, udata); if (skip < SKIP_FINI && zone->uz_fini) zone->uz_fini(item, keg->uk_size); ZONE_LOCK(zone); if (flags & ZFREE_STATFAIL) zone->uz_fails++; if (flags & ZFREE_STATFREE) zone->uz_frees++; if (!(keg->uk_flags & UMA_ZONE_MALLOC)) { mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); if (keg->uk_flags & UMA_ZONE_HASH) slab = hash_sfind(&keg->uk_hash, mem); else { mem += keg->uk_pgoff; slab = (uma_slab_t)mem; } } else { slab = (uma_slab_t)udata; } /* Do we need to remove from any lists? */ if (slab->us_freecount+1 == keg->uk_ipers) { LIST_REMOVE(slab, us_link); LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); } else if (slab->us_freecount == 0) { LIST_REMOVE(slab, us_link); LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); } /* Slab management stuff */ freei = ((unsigned long)item - (unsigned long)slab->us_data) / keg->uk_rsize; #ifdef INVARIANTS if (!skip) uma_dbg_free(zone, slab, item); #endif if (keg->uk_flags & UMA_ZONE_REFCNT) { slabref = (uma_slabrefcnt_t)slab; slabref->us_freelist[freei].us_item = slab->us_firstfree; } else { slab->us_freelist[freei].us_item = slab->us_firstfree; } slab->us_firstfree = freei; slab->us_freecount++; /* Zone statistics */ keg->uk_free++; if (keg->uk_flags & UMA_ZFLAG_FULL) { if (keg->uk_pages < keg->uk_maxpages) keg->uk_flags &= ~UMA_ZFLAG_FULL; /* * We can handle one more allocation. Since we're clearing ZFLAG_FULL, * wake up all procs blocked on pages. This should be uncommon, so * keeping this simple for now (rather than adding count of blocked * threads etc). */ wakeup(keg); } ZONE_UNLOCK(zone); }

　　uma_zfree_arg 与 uma_zalloc_arg 类似，首先尝试将 item 缓存在 当前 CPU cache 中。如失败：若 uz_free_bucket 链表不为空，则将其首元素移到 CPU cache 中以缓存 item ；否则，调用 bucket_alloc 分配一个空的 bucket ，并将之插入 uz_free_bucket 链表。如果 bucket_alloc 也失败，则调用 uma_zfree_internal 释放 item 。不过 uma_zfree_arg 在进行这些动作之前，先检查 keg 的可分配页面数是否达到上限，如是，则直接调用 uma_zfree_internal 释放 item ，因为 uma_zfree_internal 检查 keg 的可分配页面数是否已经降到上限数下了，并可唤醒阻塞在 keg 上的线程。

　　

/* * Drain the cached buckets from a zone. Expects a locked zone on entry. */ static void bucket_cache_drain(uma_zone_t zone) { uma_bucket_t bucket; /* * Drain the bucket queues and free the buckets, we just keep two per * cpu (alloc/free). */ while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { LIST_REMOVE(bucket, ub_link); ZONE_UNLOCK(zone); bucket_drain(zone, bucket); bucket_free(bucket); ZONE_LOCK(zone); } /* Now we do the free queue.. */ while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { LIST_REMOVE(bucket, ub_link); bucket_free(bucket); } } /* * Frees pages from a zone back to the system. This is done on demand from * the pageout daemon. * * Arguments: * zone The zone to free pages from * all Should we drain all items? * * Returns: * Nothing. */ void zone_drain(uma_zone_t zone) { struct slabhead freeslabs = { 0 }; uma_keg_t keg; uma_slab_t slab; uma_slab_t n; u_int8_t flags; u_int8_t *mem; int i; keg = zone->uz_keg; /* * We don't want to take pages from statically allocated zones at this * time */ if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) return; ZONE_LOCK(zone); #ifdef UMA_DEBUG printf("%s free items: %u/n", zone->uz_name, keg->uk_free); #endif bucket_cache_drain(zone); if (keg->uk_free == 0) goto finished; slab = LIST_FIRST(&keg->uk_free_slab); while (slab) { n = LIST_NEXT(slab, us_link); /* We have no where to free these to */ if (slab->us_flags & UMA_SLAB_BOOT) { slab = n; continue; } LIST_REMOVE(slab, us_link); keg->uk_pages -= keg->uk_ppera; keg->uk_free -= keg->uk_ipers; if (keg->uk_flags & UMA_ZONE_HASH) UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data); SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); slab = n; } finished: ZONE_UNLOCK(zone); while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); if (keg->uk_fini) for (i = 0; i < keg->uk_ipers; i++) keg->uk_fini( slab->us_data + (keg->uk_rsize * i), keg->uk_size); flags = slab->us_flags; mem = slab->us_data; if ((keg->uk_flags & UMA_ZONE_MALLOC) || (keg->uk_flags & UMA_ZONE_REFCNT)) { vm_object_t obj; if (flags & UMA_SLAB_KMEM) obj = kmem_object; else if (flags & UMA_SLAB_KERNEL) obj = kernel_object; else obj = NULL; for (i = 0; i < keg->uk_ppera; i++) vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), obj); } if (keg->uk_flags & UMA_ZONE_OFFPAGE) uma_zfree_internal(keg->uk_slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE); #ifdef UMA_DEBUG printf("%s: Returning %d bytes./n", zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera); #endif keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags); } }

　　zone_drain 函数通过调用 bucket_cache_drain 将 zone 中的 uz_full_bucket 缓存的 item 返还给 keg （CPU cache 中的 item 不返还），并释放 uz_full_bucket 和 uz_free_bucket 连接的所有 bucket，再将 keg 用 uk_free_slab 连接起来的 pages 释放给系统。

static void zone_foreach(void (*zfunc)(uma_zone_t)) { uma_keg_t keg; uma_zone_t zone; mtx_lock(&uma_mtx); LIST_FOREACH(keg, &uma_kegs, uk_link) { LIST_FOREACH(zone, &keg->uk_zones, uz_link) zfunc(zone); } mtx_unlock(&uma_mtx); }

　　zone_foreach 对系统中所有被 uma_kegs 连接的 zone 调用 zfunc 函数。

void uma_reclaim(void) { #ifdef UMA_DEBUG printf("UMA: vm asked us to release pages!/n"); #endif bucket_enable(); zone_foreach(zone_drain); /* * Some slabs may have been freed but this zone will be visited early * we visit again so that we can free pages that are empty once other * zones are drained. We have to do the same for buckets. */ zone_drain(slabzone); zone_drain(slabrefzone); bucket_zone_drain(); }

　　uma_reclaim 回收所有暂且不会被使用的页面，这个函数只能被 page out daemon 调用。

 

参考文章：

• Jeff Bonwick, The Slab Allocator: An Object-Caching Kernel Memory Allocator (1994)
• FreeBSD zone(9) manual page