From ca30b5db2bbf51b9c4d5aefa2ec87490b7f93395 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Tue, 17 Nov 2020 16:32:45 -0800 Subject: [PATCH] Introduce hpdata_t. Using an edata_t both for hugepages and the allocations within those hugepages was convenient at first, but has outlived its usefulness. Representing hugepages explicitly, with their own data structure, will make future development easier. --- Makefile.in | 1 + include/jemalloc/internal/edata.h | 109 ++-------- include/jemalloc/internal/hpa.h | 16 +- include/jemalloc/internal/hpdata.h | 124 +++++++++++ include/jemalloc/internal/pages.h | 14 ++ include/jemalloc/internal/psset.h | 17 +- .../projects/vc2015/jemalloc/jemalloc.vcxproj | 1 + .../vc2015/jemalloc/jemalloc.vcxproj.filters | 3 + .../projects/vc2017/jemalloc/jemalloc.vcxproj | 1 + .../vc2017/jemalloc/jemalloc.vcxproj.filters | 3 + src/edata.c | 1 - src/hpa.c | 178 +++++++--------- src/hpdata.c | 18 ++ src/pa.c | 4 +- src/psset.c | 134 ++++++------ test/unit/hpa.c | 3 +- test/unit/psset.c | 192 ++++++++---------- 17 files changed, 414 insertions(+), 405 deletions(-) create mode 100644 include/jemalloc/internal/hpdata.h create mode 100644 src/hpdata.c diff --git a/Makefile.in b/Makefile.in index eae30653..f263fc32 100644 --- a/Makefile.in +++ b/Makefile.in @@ -122,6 +122,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/hook.c \ $(srcroot)src/hpa.c \ $(srcroot)src/hpa_central.c \ + $(srcroot)src/hpdata.c \ $(srcroot)src/inspect.c \ $(srcroot)src/large.c \ $(srcroot)src/log.c \ diff --git a/include/jemalloc/internal/edata.h b/include/jemalloc/internal/edata.h index 465c962f..c0482883 100644 --- a/include/jemalloc/internal/edata.h +++ b/include/jemalloc/internal/edata.h @@ -4,6 +4,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bin_info.h" #include "jemalloc/internal/bit_util.h" +#include "jemalloc/internal/hpdata.h" #include "jemalloc/internal/nstime.h" #include "jemalloc/internal/ph.h" #include "jemalloc/internal/ql.h" @@ -71,7 +72,6 @@ struct edata_map_info_s { typedef struct edata_s edata_t; typedef ph(edata_t) edata_avail_t; typedef ph(edata_t) edata_heap_t; -typedef ph(edata_t) edata_age_heap_t; struct edata_s { /* * Bitfield containing several fields: @@ -194,41 +194,13 @@ struct edata_s { }; /* - * In some context-specific sense, the age of an active extent. Each - * context can pick a specific meaning, and share the definition of the - * edata_age_heap_t below. + * If this edata is a user allocation from an HPA, it comes out of some + * pageslab (we don't yet support huegpage allocations that don't fit + * into pageslabs). This tracks it. */ - uint64_t age; - union { - /* - * We could steal a low bit from these fields to indicate what - * sort of "thing" this is (a page slab, an object within a page - * slab, or a non-pageslab range). We don't do this yet, but it - * would enable some extra asserts. - */ - - /* - * If this edata is a user allocation from an HPA, it comes out - * of some pageslab (we don't yet support huegpage allocations - * that don't fit into pageslabs). This tracks it. - */ - edata_t *ps; - /* - * If this edata *is* a pageslab, then we cache some useful - * information about its associated bitmap. - */ - struct { - /* - * The longest free range a pageslab contains determines - * the heap it lives in. If we know that it didn't - * change after an operation, we can avoid moving it - * between heaps. - */ - uint32_t longest_free_range; - /* Whether or not the slab is backed by a hugepage. */ - bool hugeified; - }; - }; + hpdata_t *e_ps; + /* Extra field reserved for HPA. */ + void *e_reserved; union { /* @@ -330,11 +302,6 @@ edata_pai_get(const edata_t *edata) { EDATA_BITS_PAI_SHIFT); } -static inline bool -edata_hugeified_get(const edata_t *edata) { - return edata->hugeified; -} - static inline bool edata_slab_get(const edata_t *edata) { return (bool)((edata->e_bits & EDATA_BITS_SLAB_MASK) >> @@ -377,21 +344,10 @@ edata_bsize_get(const edata_t *edata) { return edata->e_bsize; } -static inline uint64_t -edata_age_get(const edata_t *edata) { - return edata->age; -} - -static inline edata_t * +static inline hpdata_t * edata_ps_get(const edata_t *edata) { assert(edata_pai_get(edata) == EXTENT_PAI_HPA); - return edata->ps; -} - -static inline uint32_t -edata_longest_free_range_get(const edata_t *edata) { - assert(edata_pai_get(edata) == EXTENT_PAI_HPA); - return edata->longest_free_range; + return edata->e_ps; } static inline void * @@ -477,21 +433,9 @@ edata_bsize_set(edata_t *edata, size_t bsize) { } static inline void -edata_age_set(edata_t *edata, uint64_t age) { - edata->age = age; -} - -static inline void -edata_ps_set(edata_t *edata, edata_t *ps) { - assert(edata_pai_get(edata) == EXTENT_PAI_HPA || ps == NULL); - edata->ps = ps; -} - -static inline void -edata_longest_free_range_set(edata_t *edata, uint32_t longest_free_range) { - assert(edata_pai_get(edata) == EXTENT_PAI_HPA - || longest_free_range == 0); - edata->longest_free_range = longest_free_range; +edata_ps_set(edata_t *edata, hpdata_t *ps) { + assert(edata_pai_get(edata) == EXTENT_PAI_HPA); + edata->e_ps = ps; } static inline void @@ -566,11 +510,6 @@ edata_pai_set(edata_t *edata, extent_pai_t pai) { ((uint64_t)pai << EDATA_BITS_PAI_SHIFT); } -static inline void -edata_hugeified_set(edata_t *edata, bool hugeified) { - edata->hugeified = hugeified; -} - static inline void edata_slab_set(edata_t *edata, bool slab) { edata->e_bits = (edata->e_bits & ~EDATA_BITS_SLAB_MASK) | @@ -633,9 +572,6 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size, if (config_prof) { edata_prof_tctx_set(edata, NULL); } - edata_age_set(edata, 0); - edata_ps_set(edata, NULL); - edata_longest_free_range_set(edata, 0); } static inline void @@ -649,15 +585,12 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, size_t sn) { edata_state_set(edata, extent_state_active); edata_zeroed_set(edata, true); edata_committed_set(edata, true); - edata_age_set(edata, 0); /* * This isn't strictly true, but base allocated extents never get * deallocated and can't be looked up in the emap, but no sense in * wasting a state bit to encode this fact. */ edata_pai_set(edata, EXTENT_PAI_PAC); - edata_ps_set(edata, NULL); - edata_longest_free_range_set(edata, 0); } static inline int @@ -718,25 +651,7 @@ edata_esnead_comp(const edata_t *a, const edata_t *b) { return ret; } -static inline int -edata_age_comp(const edata_t *a, const edata_t *b) { - uint64_t a_age = edata_age_get(a); - uint64_t b_age = edata_age_get(b); - - /* - * Equal ages are possible in certain race conditions, like two distinct - * threads simultaneously allocating a new fresh slab without holding a - * bin lock. - */ - int ret = (a_age > b_age) - (a_age < b_age); - if (ret != 0) { - return ret; - } - return edata_snad_comp(a, b); -} - ph_proto(, edata_avail_, edata_avail_t, edata_t) ph_proto(, edata_heap_, edata_heap_t, edata_t) -ph_proto(, edata_age_heap_, edata_age_heap_t, edata_t); #endif /* JEMALLOC_INTERNAL_EDATA_H */ diff --git a/include/jemalloc/internal/hpa.h b/include/jemalloc/internal/hpa.h index 1c4585df..edb36179 100644 --- a/include/jemalloc/internal/hpa.h +++ b/include/jemalloc/internal/hpa.h @@ -21,6 +21,8 @@ struct hpa_shard_s { pai_t pai; malloc_mutex_t grow_mtx; malloc_mutex_t mtx; + /* The base metadata allocator. */ + base_t *base; /* * This edata cache is the one we use when allocating a small extent * from a pageslab. The pageslab itself comes from the centralized @@ -45,7 +47,14 @@ struct hpa_shard_s { * * Guarded by grow_mtx. */ - edata_list_inactive_t unused_slabs; + hpdata_list_t unused_slabs; + + /* + * How many grow operations have occurred. + * + * Guarded by grow_mtx. + */ + uint64_t age_counter; /* * Either NULL (if empty), or some integer multiple of a @@ -54,7 +63,8 @@ struct hpa_shard_s { * * Guarded by grow_mtx. */ - edata_t *eden; + void *eden; + size_t eden_len; /* The arena ind we're associated with. */ unsigned ind; @@ -67,7 +77,7 @@ struct hpa_shard_s { * just that it can function properly given the system it's running on. */ bool hpa_supported(); -bool hpa_shard_init(hpa_shard_t *shard, emap_t *emap, +bool hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind, size_t alloc_max); void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src); diff --git a/include/jemalloc/internal/hpdata.h b/include/jemalloc/internal/hpdata.h new file mode 100644 index 00000000..c4bf6ef5 --- /dev/null +++ b/include/jemalloc/internal/hpdata.h @@ -0,0 +1,124 @@ +#ifndef JEMALLOC_INTERNAL_HPDATA_H +#define JEMALLOC_INTERNAL_HPDATA_H + +#include "jemalloc/internal/flat_bitmap.h" +#include "jemalloc/internal/ph.h" +#include "jemalloc/internal/ql.h" +#include "jemalloc/internal/typed_list.h" + +/* + * The metadata representation we use for extents in hugepages. While the PAC + * uses the edata_t to represent both active and inactive extents, the HP only + * uses the edata_t for active ones; instead, inactive extent state is tracked + * within hpdata associated with the enclosing hugepage-sized, hugepage-aligned + * region of virtual address space. + * + * An hpdata need not be "truly" backed by a hugepage (which is not necessarily + * an observable property of any given region of address space). It's just + * hugepage-sized and hugepage-aligned; it's *potentially* huge. + */ +typedef struct hpdata_s hpdata_t; +struct hpdata_s { + /* + * We likewise follow the edata convention of mangling names and forcing + * the use of accessors -- this lets us add some consistency checks on + * access. + */ + + /* + * The address of the hugepage in question. This can't be named h_addr, + * since that conflicts with a macro defined in Windows headers. + */ + void *h_address; + /* Its age (measured in psset operations). */ + uint64_t h_age; + /* Whether or not we think the hugepage is mapped that way by the OS. */ + bool h_huge; + union { + /* When nonempty, used by the psset bins. */ + phn(hpdata_t) ph_link; + /* + * When empty (or not corresponding to any hugepage), list + * linkage. + */ + ql_elm(hpdata_t) ql_link; + }; + + /* Number of currently free pages (regardless of contiguity). */ + size_t h_nfree; + /* The length of the largest contiguous sequence of inactive pages. */ + size_t h_longest_free_range; + + /* A bitmap with bits set in the active pages. */ + fb_group_t active_pages[FB_NGROUPS(HUGEPAGE_PAGES)]; +}; + +static inline void * +hpdata_addr_get(const hpdata_t *hpdata) { + return hpdata->h_address; +} + +static inline void +hpdata_addr_set(hpdata_t *hpdata, void *addr) { + assert(HUGEPAGE_ADDR2BASE(addr) == addr); + hpdata->h_address = addr; +} + +static inline uint64_t +hpdata_age_get(const hpdata_t *hpdata) { + return hpdata->h_age; +} + +static inline void +hpdata_age_set(hpdata_t *hpdata, uint64_t age) { + hpdata->h_age = age; +} + +static inline bool +hpdata_huge_get(const hpdata_t *hpdata) { + return hpdata->h_huge; +} + +static inline void +hpdata_huge_set(hpdata_t *hpdata, bool huge) { + hpdata->h_huge = huge; +} + +static inline size_t +hpdata_nfree_get(const hpdata_t *hpdata) { + return hpdata->h_nfree; +} + +static inline void +hpdata_nfree_set(hpdata_t *hpdata, size_t nfree) { + assert(nfree <= HUGEPAGE_PAGES); + hpdata->h_nfree = nfree; +} + +static inline size_t +hpdata_longest_free_range_get(const hpdata_t *hpdata) { + return hpdata->h_longest_free_range; +} + +static inline void +hpdata_longest_free_range_set(hpdata_t *hpdata, size_t longest_free_range) { + assert(longest_free_range <= HUGEPAGE_PAGES); + hpdata->h_longest_free_range = longest_free_range; +} + +static inline void +hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) { + hpdata_addr_set(hpdata, addr); + hpdata_age_set(hpdata, age); + hpdata_huge_set(hpdata, false); + hpdata_nfree_set(hpdata, HUGEPAGE_PAGES); + hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES); + fb_init(hpdata->active_pages, HUGEPAGE_PAGES); +} + +TYPED_LIST(hpdata_list, hpdata_t, ql_link) + +typedef ph(hpdata_t) hpdata_age_heap_t; +ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t); + +#endif /* JEMALLOC_INTERNAL_HPDATA_H */ diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h index cfaa0fc2..035364e2 100644 --- a/include/jemalloc/internal/pages.h +++ b/include/jemalloc/internal/pages.h @@ -17,6 +17,20 @@ /* Huge page size. LG_HUGEPAGE is determined by the configure script. */ #define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE)) #define HUGEPAGE_MASK ((size_t)(HUGEPAGE - 1)) + +#if LG_HUGEPAGE != 0 +# define HUGEPAGE_PAGES (HUGEPAGE / PAGE) +#else +/* + * It's convenient to define arrays (or bitmaps) of HUGEPAGE_PAGES lengths. If + * we can't autodetect the hugepage size, it gets treated as 0, in which case + * we'll trigger a compiler error in those arrays. Avoid this case by ensuring + * that this value is at least 1. (We won't ever run in this degraded state; + * hpa_supported() returns false in this case. + */ +# define HUGEPAGE_PAGES 1 +#endif + /* Return the huge page base address for the huge page containing address a. */ #define HUGEPAGE_ADDR2BASE(a) \ ((void *)((uintptr_t)(a) & ~HUGEPAGE_MASK)) diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index 3c9f23bb..01b4e80a 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_PSSET_H #define JEMALLOC_INTERNAL_PSSET_H +#include "jemalloc/internal/hpdata.h" + /* * A page-slab set. What the eset is to PAC, the psset is to HPA. It maintains * a collection of page-slabs (the intent being that they are backed by @@ -51,21 +53,18 @@ struct psset_s { * The pageslabs, quantized by the size class of the largest contiguous * free run of pages in a pageslab. */ - edata_age_heap_t pageslabs[PSSET_NPSIZES]; + hpdata_age_heap_t pageslabs[PSSET_NPSIZES]; bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)]; psset_stats_t stats; - - /* How many alloc_new calls have happened? */ - uint64_t age_counter; }; void psset_init(psset_t *psset); void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src); -void psset_insert(psset_t *psset, edata_t *ps); -void psset_remove(psset_t *psset, edata_t *ps); +void psset_insert(psset_t *psset, hpdata_t *ps); +void psset_remove(psset_t *psset, hpdata_t *ps); -void psset_hugify(psset_t *psset, edata_t *ps); +void psset_hugify(psset_t *psset, hpdata_t *ps); /* * Tries to obtain a chunk from an existing pageslab already in the set. @@ -78,7 +77,7 @@ bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size); * to the psset and allocate an extent from within it. The passed-in pageslab * must be at least as big as size. */ -void psset_alloc_new(psset_t *psset, edata_t *ps, +void psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size); /* @@ -89,6 +88,6 @@ void psset_alloc_new(psset_t *psset, edata_t *ps, * result must be checked and deallocated to the central HPA. Otherwise returns * NULL. */ -edata_t *psset_dalloc(psset_t *psset, edata_t *edata); +hpdata_t *psset_dalloc(psset_t *psset, edata_t *edata); #endif /* JEMALLOC_INTERNAL_PSSET_H */ diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index 6c4e7fdc..531dd9a6 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -62,6 +62,7 @@ + diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters index 84ff5748..f031fb10 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters @@ -70,6 +70,9 @@ Source Files + + Source Files + Source Files diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj index 07fbe21e..bc64de5c 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -62,6 +62,7 @@ + diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters index 84ff5748..f031fb10 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters @@ -70,6 +70,9 @@ Source Files + + Source Files + Source Files diff --git a/src/edata.c b/src/edata.c index a6597312..23523dd0 100644 --- a/src/edata.c +++ b/src/edata.c @@ -4,4 +4,3 @@ ph_gen(, edata_avail_, edata_avail_t, edata_t, ph_link, edata_esnead_comp) ph_gen(, edata_heap_, edata_heap_t, edata_t, ph_link, edata_snad_comp) -ph_gen(, edata_age_heap_, edata_age_heap_t, edata_t, ph_link, edata_age_comp) diff --git a/src/hpa.c b/src/hpa.c index ca75628c..9a190c8a 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -33,22 +33,22 @@ hpa_supported() { * We fundamentally rely on a address-space-hungry growth strategy for * hugepages. */ - if (LG_SIZEOF_PTR == 2) { + if (LG_SIZEOF_PTR != 3) { return false; } /* - * We use the edata bitmap; it needs to have at least as many bits as a - * hugepage has pages. + * If we couldn't detect the value of HUGEPAGE, HUGEPAGE_PAGES becomes + * this sentinel value -- see the comment in pages.h. */ - if (HUGEPAGE / PAGE > BITMAP_GROUPS_MAX * sizeof(bitmap_t) * 8) { + if (HUGEPAGE_PAGES == 1) { return false; } return true; } bool -hpa_shard_init(hpa_shard_t *shard, emap_t *emap, edata_cache_t *edata_cache, - unsigned ind, size_t alloc_max) { +hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base, + edata_cache_t *edata_cache, unsigned ind, size_t alloc_max) { /* malloc_conf processing should have filtered out these cases. */ assert(hpa_supported()); bool err; @@ -64,11 +64,14 @@ hpa_shard_init(hpa_shard_t *shard, emap_t *emap, edata_cache_t *edata_cache, } assert(edata_cache != NULL); + shard->base = base; edata_cache_small_init(&shard->ecs, edata_cache); psset_init(&shard->psset); shard->alloc_max = alloc_max; - edata_list_inactive_init(&shard->unused_slabs); + hpdata_list_init(&shard->unused_slabs); + shard->age_counter = 0; shard->eden = NULL; + shard->eden_len = 0; shard->ind = ind; shard->emap = emap; @@ -104,22 +107,27 @@ hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard, malloc_mutex_unlock(tsdn, &shard->mtx); } +static hpdata_t * +hpa_alloc_ps(tsdn_t *tsdn, hpa_shard_t *shard) { + return (hpdata_t *)base_alloc(tsdn, shard->base, sizeof(hpdata_t), + CACHELINE); +} + static bool -hpa_should_hugify(hpa_shard_t *shard, edata_t *ps) { +hpa_should_hugify(hpa_shard_t *shard, hpdata_t *ps) { /* * For now, just use a static check; hugify a page if it's <= 5% * inactive. Eventually, this should be a malloc conf option. */ - return !edata_hugeified_get(ps) - && edata_nfree_get(ps) < (HUGEPAGE / PAGE) * 5 / 100; + return !hpdata_huge_get(ps) + && hpdata_nfree_get(ps) < (HUGEPAGE / PAGE) * 5 / 100; } /* Returns true on error. */ static void -hpa_hugify(edata_t *ps) { - assert(edata_size_get(ps) == HUGEPAGE); - assert(edata_hugeified_get(ps)); - bool err = pages_huge(edata_base_get(ps), HUGEPAGE); +hpa_hugify(hpdata_t *ps) { + assert(hpdata_huge_get(ps)); + bool err = pages_huge(hpdata_addr_get(ps), HUGEPAGE); /* * Eat the error; even if the hugeification failed, it's still safe to * pretend it didn't (and would require extraordinary measures to @@ -129,30 +137,36 @@ hpa_hugify(edata_t *ps) { } static void -hpa_dehugify(edata_t *ps) { +hpa_dehugify(hpdata_t *ps) { /* Purge, then dehugify while unbacked. */ - pages_purge_forced(edata_addr_get(ps), HUGEPAGE); - pages_nohuge(edata_addr_get(ps), HUGEPAGE); - edata_hugeified_set(ps, false); + pages_purge_forced(hpdata_addr_get(ps), HUGEPAGE); + pages_nohuge(hpdata_addr_get(ps), HUGEPAGE); + hpdata_huge_set(ps, false); } -static edata_t * +static hpdata_t * hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) { malloc_mutex_assert_owner(tsdn, &shard->grow_mtx); - edata_t *ps = NULL; + hpdata_t *ps = NULL; /* Is there address space waiting for reuse? */ malloc_mutex_assert_owner(tsdn, &shard->grow_mtx); - ps = edata_list_inactive_first(&shard->unused_slabs); + ps = hpdata_list_first(&shard->unused_slabs); if (ps != NULL) { - edata_list_inactive_remove(&shard->unused_slabs, ps); + hpdata_list_remove(&shard->unused_slabs, ps); + hpdata_age_set(ps, shard->age_counter++); return ps; } /* Is eden a perfect fit? */ - if (shard->eden != NULL && edata_size_get(shard->eden) == HUGEPAGE) { - ps = shard->eden; + if (shard->eden != NULL && shard->eden_len == HUGEPAGE) { + ps = hpa_alloc_ps(tsdn, shard); + if (ps == NULL) { + return NULL; + } + hpdata_init(ps, shard->eden, shard->age_counter++); shard->eden = NULL; + shard->eden_len = 0; return ps; } @@ -173,78 +187,32 @@ hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) { if (new_eden == NULL) { return NULL; } - malloc_mutex_lock(tsdn, &shard->mtx); - /* Allocate ps edata, bailing if we fail. */ - ps = edata_cache_small_get(tsdn, &shard->ecs); + ps = hpa_alloc_ps(tsdn, shard); if (ps == NULL) { - malloc_mutex_unlock(tsdn, &shard->mtx); pages_unmap(new_eden, HPA_EDEN_SIZE); return NULL; } - /* Allocate eden edata, bailing if we fail. */ - shard->eden = edata_cache_small_get(tsdn, &shard->ecs); - if (shard->eden == NULL) { - edata_cache_small_put(tsdn, &shard->ecs, ps); - malloc_mutex_unlock(tsdn, &shard->mtx); - pages_unmap(new_eden, HPA_EDEN_SIZE); - return NULL; - } - /* Success. */ - malloc_mutex_unlock(tsdn, &shard->mtx); - - /* - * Note that the values here don't really make sense (e.g. eden - * is actually zeroed). But we don't use the slab metadata in - * determining subsequent allocation metadata (e.g. zero - * tracking should be done at the per-page level, not at the - * level of the hugepage). It's just a convenient data - * structure that contains much of the helpers we need (defined - * lists, a bitmap, an address field, etc.). Eventually, we'll - * have a "real" representation of a hugepage that's unconnected - * to the edata_ts it will serve allocations into. - */ - edata_init(shard->eden, shard->ind, new_eden, HPA_EDEN_SIZE, - /* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_dirty, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - /* is_head */ true); - edata_hugeified_set(shard->eden, false); + shard->eden = new_eden; + shard->eden_len = HPA_EDEN_SIZE; } else { /* Eden is already nonempty; only need an edata for ps. */ - malloc_mutex_lock(tsdn, &shard->mtx); - ps = edata_cache_small_get(tsdn, &shard->ecs); - malloc_mutex_unlock(tsdn, &shard->mtx); + ps = hpa_alloc_ps(tsdn, shard); if (ps == NULL) { return NULL; } } - /* - * We should have dropped mtx since we're not touching ecs any more, but - * we should continue to hold the grow mutex, since we're about to touch - * eden. - */ - malloc_mutex_assert_not_owner(tsdn, &shard->mtx); - malloc_mutex_assert_owner(tsdn, &shard->grow_mtx); - + assert(ps != NULL); assert(shard->eden != NULL); - assert(edata_size_get(shard->eden) > HUGEPAGE); - assert(edata_size_get(shard->eden) % HUGEPAGE == 0); - assert(edata_addr_get(shard->eden) - == HUGEPAGE_ADDR2BASE(edata_addr_get(shard->eden))); - malloc_mutex_lock(tsdn, &shard->mtx); - ps = edata_cache_small_get(tsdn, &shard->ecs); - malloc_mutex_unlock(tsdn, &shard->mtx); - if (ps == NULL) { - return NULL; - } - edata_init(ps, edata_arena_ind_get(shard->eden), - edata_addr_get(shard->eden), HUGEPAGE, /* slab */ false, - /* szind */ SC_NSIZES, /* sn */ 0, extent_state_dirty, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - /* is_head */ true); - edata_hugeified_set(ps, false); - edata_addr_set(shard->eden, edata_past_get(ps)); - edata_size_set(shard->eden, - edata_size_get(shard->eden) - HUGEPAGE); + assert(shard->eden_len > HUGEPAGE); + assert(shard->eden_len % HUGEPAGE == 0); + assert(HUGEPAGE_ADDR2BASE(shard->eden) == shard->eden); + + hpdata_init(ps, shard->eden, shard->age_counter++); + + char *eden_char = (char *)shard->eden; + eden_char += HUGEPAGE; + shard->eden = (void *)eden_char; + shard->eden_len -= HUGEPAGE; return ps; } @@ -255,7 +223,7 @@ hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) { * their address space in a list outside the psset. */ static void -hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *ps) { +hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, hpdata_t *ps) { /* * We do relatively expensive system calls. The ps was evicted, so no * one should touch it while we're also touching it. @@ -263,9 +231,6 @@ hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *ps) { malloc_mutex_assert_not_owner(tsdn, &shard->mtx); malloc_mutex_assert_not_owner(tsdn, &shard->grow_mtx); - assert(edata_size_get(ps) == HUGEPAGE); - assert(HUGEPAGE_ADDR2BASE(edata_addr_get(ps)) == edata_addr_get(ps)); - /* * We do this unconditionally, even for pages which were not originally * hugeified; it has the same effect. @@ -273,7 +238,7 @@ hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *ps) { hpa_dehugify(ps); malloc_mutex_lock(tsdn, &shard->grow_mtx); - edata_list_inactive_prepend(&shard->unused_slabs, ps); + hpdata_list_prepend(&shard->unused_slabs, ps); malloc_mutex_unlock(tsdn, &shard->grow_mtx); } @@ -307,7 +272,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom) err = emap_register_boundary(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false); if (err) { - edata_t *ps = psset_dalloc(&shard->psset, edata); + hpdata_t *ps = psset_dalloc(&shard->psset, edata); /* * The pageslab was nonempty before we started; it * should still be nonempty now, and so shouldn't get @@ -320,7 +285,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom) return NULL; } - edata_t *ps = edata_ps_get(edata); + hpdata_t *ps = edata_ps_get(edata); assert(ps != NULL); bool hugify = hpa_should_hugify(shard, ps); if (hugify) { @@ -378,16 +343,11 @@ hpa_alloc_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size) { * deallocations (and allocations of smaller sizes) may still succeed * while we're doing this potentially expensive system call. */ - edata_t *grow_edata = hpa_grow(tsdn, shard); - if (grow_edata == NULL) { + hpdata_t *grow_ps = hpa_grow(tsdn, shard); + if (grow_ps == NULL) { malloc_mutex_unlock(tsdn, &shard->grow_mtx); return NULL; } - assert(edata_arena_ind_get(grow_edata) == shard->ind); - - edata_slab_set(grow_edata, true); - fb_group_t *fb = edata_slab_data_get(grow_edata)->bitmap; - fb_init(fb, HUGEPAGE / PAGE); /* We got the new edata; allocate from it. */ malloc_mutex_lock(tsdn, &shard->mtx); @@ -395,18 +355,19 @@ hpa_alloc_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size) { if (edata == NULL) { malloc_mutex_unlock(tsdn, &shard->mtx); malloc_mutex_unlock(tsdn, &shard->grow_mtx); + hpa_handle_ps_eviction(tsdn, shard, grow_ps); return NULL; } - psset_alloc_new(&shard->psset, grow_edata, edata, size); + psset_alloc_new(&shard->psset, grow_ps, edata, size); err = emap_register_boundary(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false); if (err) { - edata_t *ps = psset_dalloc(&shard->psset, edata); + hpdata_t *ps = psset_dalloc(&shard->psset, edata); /* * The pageslab was empty except for the new allocation; it * should get evicted. */ - assert(ps == grow_edata); + assert(ps == grow_ps); edata_cache_small_put(tsdn, &shard->ecs, edata); /* * Technically the same as fallthrough at the time of this @@ -496,7 +457,7 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { assert(edata_committed_get(edata)); assert(edata_base_get(edata) != NULL); - edata_t *ps = edata_ps_get(edata); + hpdata_t *ps = edata_ps_get(edata); /* Currently, all edatas come from pageslabs. */ assert(ps != NULL); emap_deregister_boundary(tsdn, shard->emap, edata); @@ -506,7 +467,7 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { * Page slabs can move between pssets (and have their hugeified status * change) in racy ways. */ - edata_t *evicted_ps = psset_dalloc(&shard->psset, edata); + hpdata_t *evicted_ps = psset_dalloc(&shard->psset, edata); /* * If a pageslab became empty because of the dalloc, it better have been * the one we expected. @@ -562,11 +523,10 @@ hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) { hpa_assert_empty(tsdn, shard, &shard->psset); malloc_mutex_unlock(tsdn, &shard->mtx); } - edata_t *ps; - while ((ps = edata_list_inactive_first(&shard->unused_slabs)) != NULL) { - assert(edata_size_get(ps) == HUGEPAGE); - edata_list_inactive_remove(&shard->unused_slabs, ps); - pages_unmap(edata_base_get(ps), HUGEPAGE); + hpdata_t *ps; + while ((ps = hpdata_list_first(&shard->unused_slabs)) != NULL) { + hpdata_list_remove(&shard->unused_slabs, ps); + pages_unmap(hpdata_addr_get(ps), HUGEPAGE); } } diff --git a/src/hpdata.c b/src/hpdata.c new file mode 100644 index 00000000..bbe3acce --- /dev/null +++ b/src/hpdata.c @@ -0,0 +1,18 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/hpdata.h" + +static int +hpdata_age_comp(const hpdata_t *a, const hpdata_t *b) { + uint64_t a_age = hpdata_age_get(a); + uint64_t b_age = hpdata_age_get(b); + /* + * hpdata ages are operation counts in the psset; no two should be the + * same. + */ + assert(a_age != b_age); + return (a_age > b_age) - (a_age < b_age); +} + +ph_gen(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t, ph_link, hpdata_age_comp) diff --git a/src/pa.c b/src/pa.c index bc52ff43..da64b829 100644 --- a/src/pa.c +++ b/src/pa.c @@ -51,8 +51,8 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, bool pa_shard_enable_hpa(pa_shard_t *shard, size_t alloc_max, size_t sec_nshards, size_t sec_alloc_max, size_t sec_bytes_max) { - if (hpa_shard_init(&shard->hpa_shard, shard->emap, &shard->edata_cache, - shard->ind, alloc_max)) { + if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base, + &shard->edata_cache, shard->ind, alloc_max)) { return true; } if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, sec_nshards, diff --git a/src/psset.c b/src/psset.c index 2ee683b6..cebc1ce8 100644 --- a/src/psset.c +++ b/src/psset.c @@ -11,11 +11,10 @@ static const bitmap_info_t psset_bitmap_info = void psset_init(psset_t *psset) { for (unsigned i = 0; i < PSSET_NPSIZES; i++) { - edata_age_heap_new(&psset->pageslabs[i]); + hpdata_age_heap_new(&psset->pageslabs[i]); } bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true); memset(&psset->stats, 0, sizeof(psset->stats)); - psset->age_counter = 0; } static void @@ -49,18 +48,17 @@ psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) { * ensure we don't miss any heap modification operations. */ JEMALLOC_ALWAYS_INLINE void -psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, edata_t *ps, +psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps, bool insert) { - size_t *npageslabs_dst = edata_hugeified_get(ps) + size_t *npageslabs_dst = hpdata_huge_get(ps) ? &binstats->npageslabs_huge : &binstats->npageslabs_nonhuge; - size_t *nactive_dst = edata_hugeified_get(ps) + size_t *nactive_dst = hpdata_huge_get(ps) ? &binstats->nactive_huge : &binstats->nactive_nonhuge; - size_t *ninactive_dst = edata_hugeified_get(ps) + size_t *ninactive_dst = hpdata_huge_get(ps) ? &binstats->ninactive_huge : &binstats->ninactive_nonhuge; - size_t npages = edata_size_get(ps) >> LG_PAGE; - size_t ninactive = edata_nfree_get(ps); - size_t nactive = npages - ninactive; + size_t ninactive = hpdata_nfree_get(ps); + size_t nactive = HUGEPAGE_PAGES - ninactive; size_t mul = insert ? (size_t)1 : (size_t)-1; *npageslabs_dst += mul * 1; @@ -69,12 +67,12 @@ psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, edata_t *ps, } static void -psset_bin_stats_insert(psset_bin_stats_t *binstats, edata_t *ps) { +psset_bin_stats_insert(psset_bin_stats_t *binstats, hpdata_t *ps) { psset_bin_stats_insert_remove(binstats, ps, /* insert */ true); } static void -psset_bin_stats_remove(psset_bin_stats_t *binstats, edata_t *ps) { +psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) { psset_bin_stats_insert_remove(binstats, ps, /* insert */ false); } @@ -96,27 +94,27 @@ psset_bin_stats_deactivate(psset_bin_stats_t *binstats, bool huge, size_t num) { } static void -psset_edata_heap_remove(psset_t *psset, pszind_t pind, edata_t *ps) { - edata_age_heap_remove(&psset->pageslabs[pind], ps); +psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) { + hpdata_age_heap_remove(&psset->pageslabs[pind], ps); psset_bin_stats_remove(&psset->stats.nonfull_slabs[pind], ps); } static void -psset_edata_heap_insert(psset_t *psset, pszind_t pind, edata_t *ps) { - edata_age_heap_insert(&psset->pageslabs[pind], ps); +psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) { + hpdata_age_heap_insert(&psset->pageslabs[pind], ps); psset_bin_stats_insert(&psset->stats.nonfull_slabs[pind], ps); } JEMALLOC_ALWAYS_INLINE void -psset_assert_ps_consistent(edata_t *ps) { - assert(fb_urange_longest(edata_slab_data_get(ps)->bitmap, - edata_size_get(ps) >> LG_PAGE) == edata_longest_free_range_get(ps)); +psset_assert_ps_consistent(hpdata_t *ps) { + assert(fb_urange_longest(ps->active_pages, HUGEPAGE_PAGES) + == hpdata_longest_free_range_get(ps)); } void -psset_insert(psset_t *psset, edata_t *ps) { +psset_insert(psset_t *psset, hpdata_t *ps) { psset_assert_ps_consistent(ps); - size_t longest_free_range = edata_longest_free_range_get(ps); + size_t longest_free_range = hpdata_longest_free_range_get(ps); if (longest_free_range == 0) { /* @@ -131,16 +129,16 @@ psset_insert(psset_t *psset, edata_t *ps) { longest_free_range << LG_PAGE)); assert(pind < PSSET_NPSIZES); - if (edata_age_heap_empty(&psset->pageslabs[pind])) { + if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind); } - psset_edata_heap_insert(psset, pind, ps); + psset_hpdata_heap_insert(psset, pind, ps); } void -psset_remove(psset_t *psset, edata_t *ps) { +psset_remove(psset_t *psset, hpdata_t *ps) { psset_assert_ps_consistent(ps); - size_t longest_free_range = edata_longest_free_range_get(ps); + size_t longest_free_range = hpdata_longest_free_range_get(ps); if (longest_free_range == 0) { psset_bin_stats_remove(&psset->stats.full_slabs, ps); @@ -150,18 +148,18 @@ psset_remove(psset_t *psset, edata_t *ps) { pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( longest_free_range << LG_PAGE)); assert(pind < PSSET_NPSIZES); - psset_edata_heap_remove(psset, pind, ps); - if (edata_age_heap_empty(&psset->pageslabs[pind])) { + psset_hpdata_heap_remove(psset, pind, ps); + if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind); } } void -psset_hugify(psset_t *psset, edata_t *ps) { - assert(!edata_hugeified_get(ps)); +psset_hugify(psset_t *psset, hpdata_t *ps) { + assert(!hpdata_huge_get(ps)); psset_assert_ps_consistent(ps); - size_t longest_free_range = edata_longest_free_range_get(ps); + size_t longest_free_range = hpdata_longest_free_range_get(ps); psset_bin_stats_t *bin_stats; if (longest_free_range == 0) { bin_stats = &psset->stats.full_slabs; @@ -172,7 +170,7 @@ psset_hugify(psset_t *psset, edata_t *ps) { bin_stats = &psset->stats.nonfull_slabs[pind]; } psset_bin_stats_remove(bin_stats, ps); - edata_hugeified_set(ps, true); + hpdata_huge_set(ps, true); psset_bin_stats_insert(bin_stats, ps); } @@ -180,7 +178,7 @@ psset_hugify(psset_t *psset, edata_t *ps) { * Similar to PAC's extent_recycle_extract. Out of all the pageslabs in the * set, picks one that can satisfy the allocation and remove it from the set. */ -static edata_t * +static hpdata_t * psset_recycle_extract(psset_t *psset, size_t size) { pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size)); pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info, @@ -188,13 +186,13 @@ psset_recycle_extract(psset_t *psset, size_t size) { if (pind == PSSET_NPSIZES) { return NULL; } - edata_t *ps = edata_age_heap_first(&psset->pageslabs[pind]); + hpdata_t *ps = hpdata_age_heap_first(&psset->pageslabs[pind]); if (ps == NULL) { return NULL; } - psset_edata_heap_remove(psset, pind, ps); - if (edata_age_heap_empty(&psset->pageslabs[pind])) { + psset_hpdata_heap_remove(psset, pind, ps); + if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { bitmap_set(psset->bitmap, &psset_bitmap_info, pind); } @@ -207,7 +205,7 @@ psset_recycle_extract(psset_t *psset, size_t size) { * edata with a range in the pageslab, and puts ps back in the set. */ static void -psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, +psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) { size_t start = 0; /* @@ -217,15 +215,14 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, size_t begin = 0; size_t len = 0; - fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap; + fb_group_t *ps_fb = ps->active_pages; size_t npages = size >> LG_PAGE; - size_t ps_npages = edata_size_get(ps) >> LG_PAGE; size_t largest_unchosen_range = 0; while (true) { - bool found = fb_urange_iter(ps_fb, ps_npages, start, &begin, - &len); + bool found = fb_urange_iter(ps_fb, HUGEPAGE_PAGES, start, + &begin, &len); /* * A precondition to this function is that ps must be able to * serve the allocation. @@ -245,14 +242,14 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, } start = begin + len; } - uintptr_t addr = (uintptr_t)edata_base_get(ps) + begin * PAGE; + uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE; edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size, /* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active, /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA, EXTENT_NOT_HEAD); edata_ps_set(r_edata, ps); - fb_set_range(ps_fb, ps_npages, begin, npages); - edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) - npages)); + fb_set_range(ps_fb, HUGEPAGE_PAGES, begin, npages); + hpdata_nfree_set(ps, (uint32_t)(hpdata_nfree_get(ps) - npages)); /* The pageslab isn't in a bin, so no bin stats need to change. */ /* @@ -267,8 +264,8 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, * this check in the case where we're allocating from some smaller run. */ start = begin + npages; - while (start < ps_npages) { - bool found = fb_urange_iter(ps_fb, ps_npages, start, &begin, + while (start < HUGEPAGE_PAGES) { + bool found = fb_urange_iter(ps_fb, HUGEPAGE_PAGES, start, &begin, &len); if (!found) { break; @@ -278,7 +275,7 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, } start = begin + len; } - edata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range); + hpdata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range); if (largest_unchosen_range == 0) { psset_bin_stats_insert(&psset->stats.full_slabs, ps); } else { @@ -288,7 +285,7 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) { - edata_t *ps = psset_recycle_extract(psset, size); + hpdata_t *ps = psset_recycle_extract(psset, size); if (ps == NULL) { return true; } @@ -297,48 +294,43 @@ psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) { } void -psset_alloc_new(psset_t *psset, edata_t *ps, edata_t *r_edata, size_t size) { - fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap; - size_t ps_npages = edata_size_get(ps) >> LG_PAGE; - assert(fb_empty(ps_fb, ps_npages)); - assert(ps_npages >= (size >> LG_PAGE)); - edata_nfree_set(ps, (uint32_t)ps_npages); - edata_age_set(ps, psset->age_counter); - psset->age_counter++; +psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) { + fb_group_t *ps_fb = ps->active_pages; + assert(fb_empty(ps_fb, HUGEPAGE_PAGES)); + assert(hpdata_nfree_get(ps) == HUGEPAGE_PAGES); psset_ps_alloc_insert(psset, ps, r_edata, size); } -edata_t * +hpdata_t * psset_dalloc(psset_t *psset, edata_t *edata) { assert(edata_pai_get(edata) == EXTENT_PAI_HPA); assert(edata_ps_get(edata) != NULL); - edata_t *ps = edata_ps_get(edata); + hpdata_t *ps = edata_ps_get(edata); - fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap; - size_t ps_old_longest_free_range = edata_longest_free_range_get(ps); + fb_group_t *ps_fb = ps->active_pages; + size_t ps_old_longest_free_range = hpdata_longest_free_range_get(ps); pszind_t old_pind = SC_NPSIZES; if (ps_old_longest_free_range != 0) { old_pind = sz_psz2ind(sz_psz_quantize_floor( ps_old_longest_free_range << LG_PAGE)); } - size_t ps_npages = edata_size_get(ps) >> LG_PAGE; size_t begin = - ((uintptr_t)edata_base_get(edata) - (uintptr_t)edata_base_get(ps)) + ((uintptr_t)edata_base_get(edata) - (uintptr_t)hpdata_addr_get(ps)) >> LG_PAGE; size_t len = edata_size_get(edata) >> LG_PAGE; - fb_unset_range(ps_fb, ps_npages, begin, len); + fb_unset_range(ps_fb, HUGEPAGE_PAGES, begin, len); /* The pageslab is still in the bin; adjust its stats first. */ psset_bin_stats_t *bin_stats = (ps_old_longest_free_range == 0 ? &psset->stats.full_slabs : &psset->stats.nonfull_slabs[old_pind]); - psset_bin_stats_deactivate(bin_stats, edata_hugeified_get(ps), len); + psset_bin_stats_deactivate(bin_stats, hpdata_huge_get(ps), len); - edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) + len)); + hpdata_nfree_set(ps, (uint32_t)(hpdata_nfree_get(ps) + len)); /* We might have just created a new, larger range. */ - size_t new_begin = (size_t)(fb_fls(ps_fb, ps_npages, begin) + 1); - size_t new_end = fb_ffs(ps_fb, ps_npages, begin + len - 1); + size_t new_begin = (size_t)(fb_fls(ps_fb, HUGEPAGE_PAGES, begin) + 1); + size_t new_end = fb_ffs(ps_fb, HUGEPAGE_PAGES, begin + len - 1); size_t new_range_len = new_end - new_begin; /* * If the new free range is no longer than the previous longest one, @@ -352,7 +344,7 @@ psset_dalloc(psset_t *psset, edata_t *edata) { * Otherwise, it might need to get evicted from the set, or change its * bin. */ - edata_longest_free_range_set(ps, (uint32_t)new_range_len); + hpdata_longest_free_range_set(ps, (uint32_t)new_range_len); /* * If it was previously non-full, then it's in some (possibly now * incorrect) bin already; remove it. @@ -366,8 +358,8 @@ psset_dalloc(psset_t *psset, edata_t *edata) { * and the issue becomes moot). */ if (ps_old_longest_free_range > 0) { - psset_edata_heap_remove(psset, old_pind, ps); - if (edata_age_heap_empty(&psset->pageslabs[old_pind])) { + psset_hpdata_heap_remove(psset, old_pind, ps); + if (hpdata_age_heap_empty(&psset->pageslabs[old_pind])) { bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)old_pind); } @@ -379,16 +371,16 @@ psset_dalloc(psset_t *psset, edata_t *edata) { psset_bin_stats_remove(&psset->stats.full_slabs, ps); } /* If the pageslab is empty, it gets evicted from the set. */ - if (new_range_len == ps_npages) { + if (new_range_len == HUGEPAGE_PAGES) { return ps; } /* Otherwise, it gets reinserted. */ pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor( new_range_len << LG_PAGE)); - if (edata_age_heap_empty(&psset->pageslabs[new_pind])) { + if (hpdata_age_heap_empty(&psset->pageslabs[new_pind])) { bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)new_pind); } - psset_edata_heap_insert(psset, new_pind, ps); + psset_hpdata_heap_insert(psset, new_pind, ps); return NULL; } diff --git a/test/unit/hpa.c b/test/unit/hpa.c index 94efd4ae..90ec89e4 100644 --- a/test/unit/hpa.c +++ b/test/unit/hpa.c @@ -38,7 +38,8 @@ create_test_data() { assert_false(err, ""); err = hpa_shard_init(&test_data->shard, &test_data->emap, - &test_data->shard_edata_cache, SHARD_IND, ALLOC_MAX); + test_data->base, &test_data->shard_edata_cache, SHARD_IND, + ALLOC_MAX); assert_false(err, ""); return (hpa_shard_t *)test_data; diff --git a/test/unit/psset.c b/test/unit/psset.c index ea61ab92..811c7be1 100644 --- a/test/unit/psset.c +++ b/test/unit/psset.c @@ -2,10 +2,8 @@ #include "jemalloc/internal/psset.h" -#define PAGESLAB_PAGES (HUGEPAGE / PAGE) -#define PAGESLAB_SIZE (PAGESLAB_PAGES << LG_PAGE) -#define PAGESLAB_SN 123 -#define PAGESLAB_ADDR ((void *)(1234 << LG_PAGE)) +#define PAGESLAB_ADDR ((void *)(1234 * HUGEPAGE)) +#define PAGESLAB_AGE 5678 #define ALLOC_ARENA_IND 111 #define ALLOC_ESN 222 @@ -42,14 +40,10 @@ edata_expect(edata_t *edata, size_t page_offset, size_t page_cnt) { TEST_BEGIN(test_empty) { bool err; - edata_t pageslab; - memset(&pageslab, 0, sizeof(pageslab)); - edata_t alloc; + hpdata_t pageslab; + hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE); - edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE, - /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - EXTENT_IS_HEAD); + edata_t alloc; edata_init_test(&alloc); psset_t psset; @@ -63,27 +57,24 @@ TEST_END TEST_BEGIN(test_fill) { bool err; - edata_t pageslab; - memset(&pageslab, 0, sizeof(pageslab)); - edata_t alloc[PAGESLAB_PAGES]; - edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE, - /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - EXTENT_IS_HEAD); + hpdata_t pageslab; + hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE); + + edata_t alloc[HUGEPAGE_PAGES]; psset_t psset; psset_init(&psset); edata_init_test(&alloc[0]); psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE); - for (size_t i = 1; i < PAGESLAB_PAGES; i++) { + for (size_t i = 1; i < HUGEPAGE_PAGES; i++) { edata_init_test(&alloc[i]); err = psset_alloc_reuse(&psset, &alloc[i], PAGE); expect_false(err, "Nonempty psset failed page allocation."); } - for (size_t i = 0; i < PAGESLAB_PAGES; i++) { + for (size_t i = 0; i < HUGEPAGE_PAGES; i++) { edata_t *edata = &alloc[i]; edata_expect(edata, i, 1); } @@ -98,30 +89,26 @@ TEST_END TEST_BEGIN(test_reuse) { bool err; - edata_t *ps; + hpdata_t *ps; - edata_t pageslab; - memset(&pageslab, 0, sizeof(pageslab)); - edata_t alloc[PAGESLAB_PAGES]; + hpdata_t pageslab; + hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE); - edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE, - /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - EXTENT_IS_HEAD); + edata_t alloc[HUGEPAGE_PAGES]; psset_t psset; psset_init(&psset); edata_init_test(&alloc[0]); psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE); - for (size_t i = 1; i < PAGESLAB_PAGES; i++) { + for (size_t i = 1; i < HUGEPAGE_PAGES; i++) { edata_init_test(&alloc[i]); err = psset_alloc_reuse(&psset, &alloc[i], PAGE); expect_false(err, "Nonempty psset failed page allocation."); } /* Free odd indices. */ - for (size_t i = 0; i < PAGESLAB_PAGES; i ++) { + for (size_t i = 0; i < HUGEPAGE_PAGES; i ++) { if (i % 2 == 0) { continue; } @@ -129,7 +116,7 @@ TEST_BEGIN(test_reuse) { expect_ptr_null(ps, "Nonempty pageslab evicted"); } /* Realloc into them. */ - for (size_t i = 0; i < PAGESLAB_PAGES; i++) { + for (size_t i = 0; i < HUGEPAGE_PAGES; i++) { if (i % 2 == 0) { continue; } @@ -138,7 +125,7 @@ TEST_BEGIN(test_reuse) { edata_expect(&alloc[i], i, 1); } /* Now, free the pages at indices 0 or 1 mod 2. */ - for (size_t i = 0; i < PAGESLAB_PAGES; i++) { + for (size_t i = 0; i < HUGEPAGE_PAGES; i++) { if (i % 4 > 1) { continue; } @@ -146,7 +133,7 @@ TEST_BEGIN(test_reuse) { expect_ptr_null(ps, "Nonempty pageslab evicted"); } /* And realloc 2-page allocations into them. */ - for (size_t i = 0; i < PAGESLAB_PAGES; i++) { + for (size_t i = 0; i < HUGEPAGE_PAGES; i++) { if (i % 4 != 0) { continue; } @@ -155,7 +142,7 @@ TEST_BEGIN(test_reuse) { edata_expect(&alloc[i], i, 2); } /* Free all the 2-page allocations. */ - for (size_t i = 0; i < PAGESLAB_PAGES; i++) { + for (size_t i = 0; i < HUGEPAGE_PAGES; i++) { if (i % 4 != 0) { continue; } @@ -175,13 +162,13 @@ TEST_BEGIN(test_reuse) { edata_expect(&alloc[index_of_3], index_of_3, 3); /* Free up a 4-page hole at the end. */ - ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]); + ps = psset_dalloc(&psset, &alloc[HUGEPAGE_PAGES - 1]); expect_ptr_null(ps, "Nonempty pageslab evicted"); - ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 2]); + ps = psset_dalloc(&psset, &alloc[HUGEPAGE_PAGES - 2]); expect_ptr_null(ps, "Nonempty pageslab evicted"); /* Make sure we can satisfy an allocation at the very end of a slab. */ - size_t index_of_4 = PAGESLAB_PAGES - 4; + size_t index_of_4 = HUGEPAGE_PAGES - 4; ps = psset_dalloc(&psset, &alloc[index_of_4]); expect_ptr_null(ps, "Nonempty pageslab evicted"); err = psset_alloc_reuse(&psset, &alloc[index_of_4], 4 * PAGE); @@ -192,33 +179,31 @@ TEST_END TEST_BEGIN(test_evict) { bool err; - edata_t *ps; - edata_t pageslab; - memset(&pageslab, 0, sizeof(pageslab)); - edata_t alloc[PAGESLAB_PAGES]; + hpdata_t *ps; + + hpdata_t pageslab; + hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE); + + edata_t alloc[HUGEPAGE_PAGES]; - edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE, - /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - EXTENT_IS_HEAD); psset_t psset; psset_init(&psset); /* Alloc the whole slab. */ edata_init_test(&alloc[0]); psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE); - for (size_t i = 1; i < PAGESLAB_PAGES; i++) { + for (size_t i = 1; i < HUGEPAGE_PAGES; i++) { edata_init_test(&alloc[i]); err = psset_alloc_reuse(&psset, &alloc[i], PAGE); expect_false(err, "Unxpected allocation failure"); } /* Dealloc the whole slab, going forwards. */ - for (size_t i = 0; i < PAGESLAB_PAGES - 1; i++) { + for (size_t i = 0; i < HUGEPAGE_PAGES - 1; i++) { ps = psset_dalloc(&psset, &alloc[i]); expect_ptr_null(ps, "Nonempty pageslab evicted"); } - ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]); + ps = psset_dalloc(&psset, &alloc[HUGEPAGE_PAGES - 1]); expect_ptr_eq(&pageslab, ps, "Empty pageslab not evicted."); err = psset_alloc_reuse(&psset, &alloc[0], PAGE); @@ -228,20 +213,15 @@ TEST_END TEST_BEGIN(test_multi_pageslab) { bool err; - edata_t *ps; - edata_t pageslab[2]; - memset(&pageslab, 0, sizeof(pageslab)); - edata_t alloc[2][PAGESLAB_PAGES]; + hpdata_t *ps; - edata_init(&pageslab[0], /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE, - /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - EXTENT_IS_HEAD); - edata_init(&pageslab[1], /* arena_ind */ 0, - (void *)((uintptr_t)PAGESLAB_ADDR + PAGESLAB_SIZE), PAGESLAB_SIZE, - /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - EXTENT_IS_HEAD); + hpdata_t pageslab[2]; + hpdata_init(&pageslab[0], PAGESLAB_ADDR, PAGESLAB_AGE); + hpdata_init(&pageslab[1], + (void *)((uintptr_t)PAGESLAB_ADDR + HUGEPAGE), + PAGESLAB_AGE + 1); + + edata_t alloc[2][HUGEPAGE_PAGES]; psset_t psset; psset_init(&psset); @@ -254,7 +234,7 @@ TEST_BEGIN(test_multi_pageslab) { /* Fill them both up; make sure we do so in first-fit order. */ for (size_t i = 0; i < 2; i++) { - for (size_t j = 1; j < PAGESLAB_PAGES; j++) { + for (size_t j = 1; j < HUGEPAGE_PAGES; j++) { edata_init_test(&alloc[i][j]); err = psset_alloc_reuse(&psset, &alloc[i][j], PAGE); expect_false(err, @@ -306,10 +286,10 @@ stats_expect_empty(psset_bin_stats_t *stats) { static void stats_expect(psset_t *psset, size_t nactive) { - if (nactive == PAGESLAB_PAGES) { + if (nactive == HUGEPAGE_PAGES) { expect_zu_eq(1, psset->stats.full_slabs.npageslabs_nonhuge, "Expected a full slab"); - expect_zu_eq(PAGESLAB_PAGES, + expect_zu_eq(HUGEPAGE_PAGES, psset->stats.full_slabs.nactive_nonhuge, "Should have exactly filled the bin"); expect_zu_eq(0, psset->stats.full_slabs.ninactive_nonhuge, @@ -317,9 +297,9 @@ stats_expect(psset_t *psset, size_t nactive) { } else { stats_expect_empty(&psset->stats.full_slabs); } - size_t ninactive = PAGESLAB_PAGES - nactive; + size_t ninactive = HUGEPAGE_PAGES - nactive; pszind_t nonempty_pind = PSSET_NPSIZES; - if (ninactive != 0 && ninactive < PAGESLAB_PAGES) { + if (ninactive != 0 && ninactive < HUGEPAGE_PAGES) { nonempty_pind = sz_psz2ind(sz_psz_quantize_floor( ninactive << LG_PAGE)); } @@ -342,14 +322,11 @@ stats_expect(psset_t *psset, size_t nactive) { TEST_BEGIN(test_stats) { bool err; - edata_t pageslab; - memset(&pageslab, 0, sizeof(pageslab)); - edata_t alloc[PAGESLAB_PAGES]; - edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE, - /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active, - /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, - EXTENT_IS_HEAD); + hpdata_t pageslab; + hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE); + + edata_t alloc[HUGEPAGE_PAGES]; psset_t psset; psset_init(&psset); @@ -357,15 +334,15 @@ TEST_BEGIN(test_stats) { edata_init_test(&alloc[0]); psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE); - for (size_t i = 1; i < PAGESLAB_PAGES; i++) { + for (size_t i = 1; i < HUGEPAGE_PAGES; i++) { stats_expect(&psset, i); edata_init_test(&alloc[i]); err = psset_alloc_reuse(&psset, &alloc[i], PAGE); expect_false(err, "Nonempty psset failed page allocation."); } - stats_expect(&psset, PAGESLAB_PAGES); - edata_t *ps; - for (ssize_t i = PAGESLAB_PAGES - 1; i >= 0; i--) { + stats_expect(&psset, HUGEPAGE_PAGES); + hpdata_t *ps; + for (ssize_t i = HUGEPAGE_PAGES - 1; i >= 0; i--) { ps = psset_dalloc(&psset, &alloc[i]); expect_true((ps == NULL) == (i != 0), "psset_dalloc should only evict a slab on the last free"); @@ -384,37 +361,28 @@ TEST_END /* * Fills in and inserts two pageslabs, with the first better than the second, * and each fully allocated (into the allocations in allocs and worse_allocs, - * each of which should be PAGESLAB_PAGES long). + * each of which should be HUGEPAGE_PAGES long). * * (There's nothing magic about these numbers; it's just useful to share the * setup between the oldest fit and the insert/remove test). */ static void -init_test_pageslabs(psset_t *psset, edata_t *pageslab, edata_t *worse_pageslab, - edata_t *alloc, edata_t *worse_alloc) { +init_test_pageslabs(psset_t *psset, hpdata_t *pageslab, + hpdata_t *worse_pageslab, edata_t *alloc, edata_t *worse_alloc) { bool err; - memset(pageslab, 0, sizeof(*pageslab)); - edata_init(pageslab, /* arena_ind */ 0, (void *)(10 * PAGESLAB_SIZE), - PAGESLAB_SIZE, /* slab */ true, SC_NSIZES, PAGESLAB_SN + 1, - extent_state_active, /* zeroed */ false, /* comitted */ true, - EXTENT_PAI_HPA, EXTENT_IS_HEAD); + hpdata_init(pageslab, (void *)(10 * HUGEPAGE), PAGESLAB_AGE); /* - * This pageslab is better from an edata_comp_snad POV, but will be - * added to the set after the previous one, and so should be less - * preferred for allocations. + * This pageslab would be better from an address-first-fit POV, but + * better from an age POV. */ - memset(worse_pageslab, 0, sizeof(*worse_pageslab)); - edata_init(worse_pageslab, /* arena_ind */ 0, - (void *)(9 * PAGESLAB_SIZE), PAGESLAB_SIZE, /* slab */ true, - SC_NSIZES, PAGESLAB_SN - 1, extent_state_active, /* zeroed */ false, - /* comitted */ true, EXTENT_PAI_HPA, EXTENT_IS_HEAD); + hpdata_init(worse_pageslab, (void *)(9 * HUGEPAGE), PAGESLAB_AGE + 1); psset_init(psset); edata_init_test(&alloc[0]); psset_alloc_new(psset, pageslab, &alloc[0], PAGE); - for (size_t i = 1; i < PAGESLAB_PAGES; i++) { + for (size_t i = 1; i < HUGEPAGE_PAGES; i++) { edata_init_test(&alloc[i]); err = psset_alloc_reuse(psset, &alloc[i], PAGE); expect_false(err, "Nonempty psset failed page allocation."); @@ -430,7 +398,7 @@ init_test_pageslabs(psset_t *psset, edata_t *pageslab, edata_t *worse_pageslab, * Make the two pssets otherwise indistinguishable; all full except for * a single page. */ - for (size_t i = 1; i < PAGESLAB_PAGES - 1; i++) { + for (size_t i = 1; i < HUGEPAGE_PAGES - 1; i++) { edata_init_test(&worse_alloc[i]); err = psset_alloc_reuse(psset, &alloc[i], PAGE); expect_false(err, "Nonempty psset failed page allocation."); @@ -439,17 +407,17 @@ init_test_pageslabs(psset_t *psset, edata_t *pageslab, edata_t *worse_pageslab, } /* Deallocate the last page from the older pageslab. */ - edata_t *evicted = psset_dalloc(psset, &alloc[PAGESLAB_PAGES - 1]); + hpdata_t *evicted = psset_dalloc(psset, &alloc[HUGEPAGE_PAGES - 1]); expect_ptr_null(evicted, "Unexpected eviction"); } TEST_BEGIN(test_oldest_fit) { bool err; - edata_t alloc[PAGESLAB_PAGES]; - edata_t worse_alloc[PAGESLAB_PAGES]; + edata_t alloc[HUGEPAGE_PAGES]; + edata_t worse_alloc[HUGEPAGE_PAGES]; - edata_t pageslab; - edata_t worse_pageslab; + hpdata_t pageslab; + hpdata_t worse_pageslab; psset_t psset; @@ -468,12 +436,12 @@ TEST_END TEST_BEGIN(test_insert_remove) { bool err; - edata_t *ps; - edata_t alloc[PAGESLAB_PAGES]; - edata_t worse_alloc[PAGESLAB_PAGES]; + hpdata_t *ps; + edata_t alloc[HUGEPAGE_PAGES]; + edata_t worse_alloc[HUGEPAGE_PAGES]; - edata_t pageslab; - edata_t worse_pageslab; + hpdata_t pageslab; + hpdata_t worse_pageslab; psset_t psset; @@ -482,31 +450,31 @@ TEST_BEGIN(test_insert_remove) { /* Remove better; should still be able to alloc from worse. */ psset_remove(&psset, &pageslab); - err = psset_alloc_reuse(&psset, &worse_alloc[PAGESLAB_PAGES - 1], PAGE); + err = psset_alloc_reuse(&psset, &worse_alloc[HUGEPAGE_PAGES - 1], PAGE); expect_false(err, "Removal should still leave an empty page"); expect_ptr_eq(&worse_pageslab, - edata_ps_get(&worse_alloc[PAGESLAB_PAGES - 1]), + edata_ps_get(&worse_alloc[HUGEPAGE_PAGES - 1]), "Allocated out of wrong ps"); /* * After deallocating the previous alloc and reinserting better, it * should be preferred for future allocations. */ - ps = psset_dalloc(&psset, &worse_alloc[PAGESLAB_PAGES - 1]); + ps = psset_dalloc(&psset, &worse_alloc[HUGEPAGE_PAGES - 1]); expect_ptr_null(ps, "Incorrect eviction of nonempty pageslab"); psset_insert(&psset, &pageslab); - err = psset_alloc_reuse(&psset, &alloc[PAGESLAB_PAGES - 1], PAGE); + err = psset_alloc_reuse(&psset, &alloc[HUGEPAGE_PAGES - 1], PAGE); expect_false(err, "psset should be nonempty"); - expect_ptr_eq(&pageslab, edata_ps_get(&alloc[PAGESLAB_PAGES - 1]), + expect_ptr_eq(&pageslab, edata_ps_get(&alloc[HUGEPAGE_PAGES - 1]), "Removal/reinsertion shouldn't change ordering"); /* * After deallocating and removing both, allocations should fail. */ - ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]); + ps = psset_dalloc(&psset, &alloc[HUGEPAGE_PAGES - 1]); expect_ptr_null(ps, "Incorrect eviction"); psset_remove(&psset, &pageslab); psset_remove(&psset, &worse_pageslab); - err = psset_alloc_reuse(&psset, &alloc[PAGESLAB_PAGES - 1], PAGE); + err = psset_alloc_reuse(&psset, &alloc[HUGEPAGE_PAGES - 1], PAGE); expect_true(err, "psset should be empty, but an alloc succeeded"); } TEST_END