diff --git a/Makefile.in b/Makefile.in
index eae30653..f263fc32 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -122,6 +122,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/hook.c \
$(srcroot)src/hpa.c \
$(srcroot)src/hpa_central.c \
+ $(srcroot)src/hpdata.c \
$(srcroot)src/inspect.c \
$(srcroot)src/large.c \
$(srcroot)src/log.c \
diff --git a/include/jemalloc/internal/edata.h b/include/jemalloc/internal/edata.h
index 465c962f..c0482883 100644
--- a/include/jemalloc/internal/edata.h
+++ b/include/jemalloc/internal/edata.h
@@ -4,6 +4,7 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bin_info.h"
#include "jemalloc/internal/bit_util.h"
+#include "jemalloc/internal/hpdata.h"
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/ph.h"
#include "jemalloc/internal/ql.h"
@@ -71,7 +72,6 @@ struct edata_map_info_s {
typedef struct edata_s edata_t;
typedef ph(edata_t) edata_avail_t;
typedef ph(edata_t) edata_heap_t;
-typedef ph(edata_t) edata_age_heap_t;
struct edata_s {
/*
* Bitfield containing several fields:
@@ -194,41 +194,13 @@ struct edata_s {
};
/*
- * In some context-specific sense, the age of an active extent. Each
- * context can pick a specific meaning, and share the definition of the
- * edata_age_heap_t below.
+ * If this edata is a user allocation from an HPA, it comes out of some
+ * pageslab (we don't yet support huegpage allocations that don't fit
+ * into pageslabs). This tracks it.
*/
- uint64_t age;
- union {
- /*
- * We could steal a low bit from these fields to indicate what
- * sort of "thing" this is (a page slab, an object within a page
- * slab, or a non-pageslab range). We don't do this yet, but it
- * would enable some extra asserts.
- */
-
- /*
- * If this edata is a user allocation from an HPA, it comes out
- * of some pageslab (we don't yet support huegpage allocations
- * that don't fit into pageslabs). This tracks it.
- */
- edata_t *ps;
- /*
- * If this edata *is* a pageslab, then we cache some useful
- * information about its associated bitmap.
- */
- struct {
- /*
- * The longest free range a pageslab contains determines
- * the heap it lives in. If we know that it didn't
- * change after an operation, we can avoid moving it
- * between heaps.
- */
- uint32_t longest_free_range;
- /* Whether or not the slab is backed by a hugepage. */
- bool hugeified;
- };
- };
+ hpdata_t *e_ps;
+ /* Extra field reserved for HPA. */
+ void *e_reserved;
union {
/*
@@ -330,11 +302,6 @@ edata_pai_get(const edata_t *edata) {
EDATA_BITS_PAI_SHIFT);
}
-static inline bool
-edata_hugeified_get(const edata_t *edata) {
- return edata->hugeified;
-}
-
static inline bool
edata_slab_get(const edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_SLAB_MASK) >>
@@ -377,21 +344,10 @@ edata_bsize_get(const edata_t *edata) {
return edata->e_bsize;
}
-static inline uint64_t
-edata_age_get(const edata_t *edata) {
- return edata->age;
-}
-
-static inline edata_t *
+static inline hpdata_t *
edata_ps_get(const edata_t *edata) {
assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
- return edata->ps;
-}
-
-static inline uint32_t
-edata_longest_free_range_get(const edata_t *edata) {
- assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
- return edata->longest_free_range;
+ return edata->e_ps;
}
static inline void *
@@ -477,21 +433,9 @@ edata_bsize_set(edata_t *edata, size_t bsize) {
}
static inline void
-edata_age_set(edata_t *edata, uint64_t age) {
- edata->age = age;
-}
-
-static inline void
-edata_ps_set(edata_t *edata, edata_t *ps) {
- assert(edata_pai_get(edata) == EXTENT_PAI_HPA || ps == NULL);
- edata->ps = ps;
-}
-
-static inline void
-edata_longest_free_range_set(edata_t *edata, uint32_t longest_free_range) {
- assert(edata_pai_get(edata) == EXTENT_PAI_HPA
- || longest_free_range == 0);
- edata->longest_free_range = longest_free_range;
+edata_ps_set(edata_t *edata, hpdata_t *ps) {
+ assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
+ edata->e_ps = ps;
}
static inline void
@@ -566,11 +510,6 @@ edata_pai_set(edata_t *edata, extent_pai_t pai) {
((uint64_t)pai << EDATA_BITS_PAI_SHIFT);
}
-static inline void
-edata_hugeified_set(edata_t *edata, bool hugeified) {
- edata->hugeified = hugeified;
-}
-
static inline void
edata_slab_set(edata_t *edata, bool slab) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_SLAB_MASK) |
@@ -633,9 +572,6 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
if (config_prof) {
edata_prof_tctx_set(edata, NULL);
}
- edata_age_set(edata, 0);
- edata_ps_set(edata, NULL);
- edata_longest_free_range_set(edata, 0);
}
static inline void
@@ -649,15 +585,12 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, size_t sn) {
edata_state_set(edata, extent_state_active);
edata_zeroed_set(edata, true);
edata_committed_set(edata, true);
- edata_age_set(edata, 0);
/*
* This isn't strictly true, but base allocated extents never get
* deallocated and can't be looked up in the emap, but no sense in
* wasting a state bit to encode this fact.
*/
edata_pai_set(edata, EXTENT_PAI_PAC);
- edata_ps_set(edata, NULL);
- edata_longest_free_range_set(edata, 0);
}
static inline int
@@ -718,25 +651,7 @@ edata_esnead_comp(const edata_t *a, const edata_t *b) {
return ret;
}
-static inline int
-edata_age_comp(const edata_t *a, const edata_t *b) {
- uint64_t a_age = edata_age_get(a);
- uint64_t b_age = edata_age_get(b);
-
- /*
- * Equal ages are possible in certain race conditions, like two distinct
- * threads simultaneously allocating a new fresh slab without holding a
- * bin lock.
- */
- int ret = (a_age > b_age) - (a_age < b_age);
- if (ret != 0) {
- return ret;
- }
- return edata_snad_comp(a, b);
-}
-
ph_proto(, edata_avail_, edata_avail_t, edata_t)
ph_proto(, edata_heap_, edata_heap_t, edata_t)
-ph_proto(, edata_age_heap_, edata_age_heap_t, edata_t);
#endif /* JEMALLOC_INTERNAL_EDATA_H */
diff --git a/include/jemalloc/internal/hpa.h b/include/jemalloc/internal/hpa.h
index 1c4585df..edb36179 100644
--- a/include/jemalloc/internal/hpa.h
+++ b/include/jemalloc/internal/hpa.h
@@ -21,6 +21,8 @@ struct hpa_shard_s {
pai_t pai;
malloc_mutex_t grow_mtx;
malloc_mutex_t mtx;
+ /* The base metadata allocator. */
+ base_t *base;
/*
* This edata cache is the one we use when allocating a small extent
* from a pageslab. The pageslab itself comes from the centralized
@@ -45,7 +47,14 @@ struct hpa_shard_s {
*
* Guarded by grow_mtx.
*/
- edata_list_inactive_t unused_slabs;
+ hpdata_list_t unused_slabs;
+
+ /*
+ * How many grow operations have occurred.
+ *
+ * Guarded by grow_mtx.
+ */
+ uint64_t age_counter;
/*
* Either NULL (if empty), or some integer multiple of a
@@ -54,7 +63,8 @@ struct hpa_shard_s {
*
* Guarded by grow_mtx.
*/
- edata_t *eden;
+ void *eden;
+ size_t eden_len;
/* The arena ind we're associated with. */
unsigned ind;
@@ -67,7 +77,7 @@ struct hpa_shard_s {
* just that it can function properly given the system it's running on.
*/
bool hpa_supported();
-bool hpa_shard_init(hpa_shard_t *shard, emap_t *emap,
+bool hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
edata_cache_t *edata_cache, unsigned ind, size_t alloc_max);
void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
diff --git a/include/jemalloc/internal/hpdata.h b/include/jemalloc/internal/hpdata.h
new file mode 100644
index 00000000..c4bf6ef5
--- /dev/null
+++ b/include/jemalloc/internal/hpdata.h
@@ -0,0 +1,124 @@
+#ifndef JEMALLOC_INTERNAL_HPDATA_H
+#define JEMALLOC_INTERNAL_HPDATA_H
+
+#include "jemalloc/internal/flat_bitmap.h"
+#include "jemalloc/internal/ph.h"
+#include "jemalloc/internal/ql.h"
+#include "jemalloc/internal/typed_list.h"
+
+/*
+ * The metadata representation we use for extents in hugepages. While the PAC
+ * uses the edata_t to represent both active and inactive extents, the HP only
+ * uses the edata_t for active ones; instead, inactive extent state is tracked
+ * within hpdata associated with the enclosing hugepage-sized, hugepage-aligned
+ * region of virtual address space.
+ *
+ * An hpdata need not be "truly" backed by a hugepage (which is not necessarily
+ * an observable property of any given region of address space). It's just
+ * hugepage-sized and hugepage-aligned; it's *potentially* huge.
+ */
+typedef struct hpdata_s hpdata_t;
+struct hpdata_s {
+ /*
+ * We likewise follow the edata convention of mangling names and forcing
+ * the use of accessors -- this lets us add some consistency checks on
+ * access.
+ */
+
+ /*
+ * The address of the hugepage in question. This can't be named h_addr,
+ * since that conflicts with a macro defined in Windows headers.
+ */
+ void *h_address;
+ /* Its age (measured in psset operations). */
+ uint64_t h_age;
+ /* Whether or not we think the hugepage is mapped that way by the OS. */
+ bool h_huge;
+ union {
+ /* When nonempty, used by the psset bins. */
+ phn(hpdata_t) ph_link;
+ /*
+ * When empty (or not corresponding to any hugepage), list
+ * linkage.
+ */
+ ql_elm(hpdata_t) ql_link;
+ };
+
+ /* Number of currently free pages (regardless of contiguity). */
+ size_t h_nfree;
+ /* The length of the largest contiguous sequence of inactive pages. */
+ size_t h_longest_free_range;
+
+ /* A bitmap with bits set in the active pages. */
+ fb_group_t active_pages[FB_NGROUPS(HUGEPAGE_PAGES)];
+};
+
+static inline void *
+hpdata_addr_get(const hpdata_t *hpdata) {
+ return hpdata->h_address;
+}
+
+static inline void
+hpdata_addr_set(hpdata_t *hpdata, void *addr) {
+ assert(HUGEPAGE_ADDR2BASE(addr) == addr);
+ hpdata->h_address = addr;
+}
+
+static inline uint64_t
+hpdata_age_get(const hpdata_t *hpdata) {
+ return hpdata->h_age;
+}
+
+static inline void
+hpdata_age_set(hpdata_t *hpdata, uint64_t age) {
+ hpdata->h_age = age;
+}
+
+static inline bool
+hpdata_huge_get(const hpdata_t *hpdata) {
+ return hpdata->h_huge;
+}
+
+static inline void
+hpdata_huge_set(hpdata_t *hpdata, bool huge) {
+ hpdata->h_huge = huge;
+}
+
+static inline size_t
+hpdata_nfree_get(const hpdata_t *hpdata) {
+ return hpdata->h_nfree;
+}
+
+static inline void
+hpdata_nfree_set(hpdata_t *hpdata, size_t nfree) {
+ assert(nfree <= HUGEPAGE_PAGES);
+ hpdata->h_nfree = nfree;
+}
+
+static inline size_t
+hpdata_longest_free_range_get(const hpdata_t *hpdata) {
+ return hpdata->h_longest_free_range;
+}
+
+static inline void
+hpdata_longest_free_range_set(hpdata_t *hpdata, size_t longest_free_range) {
+ assert(longest_free_range <= HUGEPAGE_PAGES);
+ hpdata->h_longest_free_range = longest_free_range;
+}
+
+static inline void
+hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
+ hpdata_addr_set(hpdata, addr);
+ hpdata_age_set(hpdata, age);
+ hpdata_huge_set(hpdata, false);
+ hpdata_nfree_set(hpdata, HUGEPAGE_PAGES);
+ hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES);
+ fb_init(hpdata->active_pages, HUGEPAGE_PAGES);
+}
+
+TYPED_LIST(hpdata_list, hpdata_t, ql_link)
+
+typedef ph(hpdata_t) hpdata_age_heap_t;
+ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t);
+
+#endif /* JEMALLOC_INTERNAL_HPDATA_H */
diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h
index cfaa0fc2..035364e2 100644
--- a/include/jemalloc/internal/pages.h
+++ b/include/jemalloc/internal/pages.h
@@ -17,6 +17,20 @@
/* Huge page size. LG_HUGEPAGE is determined by the configure script. */
#define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE))
#define HUGEPAGE_MASK ((size_t)(HUGEPAGE - 1))
+
+#if LG_HUGEPAGE != 0
+# define HUGEPAGE_PAGES (HUGEPAGE / PAGE)
+#else
+/*
+ * It's convenient to define arrays (or bitmaps) of HUGEPAGE_PAGES lengths. If
+ * we can't autodetect the hugepage size, it gets treated as 0, in which case
+ * we'll trigger a compiler error in those arrays. Avoid this case by ensuring
+ * that this value is at least 1. (We won't ever run in this degraded state;
+ * hpa_supported() returns false in this case.
+ */
+# define HUGEPAGE_PAGES 1
+#endif
+
/* Return the huge page base address for the huge page containing address a. */
#define HUGEPAGE_ADDR2BASE(a) \
((void *)((uintptr_t)(a) & ~HUGEPAGE_MASK))
diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h
index 3c9f23bb..01b4e80a 100644
--- a/include/jemalloc/internal/psset.h
+++ b/include/jemalloc/internal/psset.h
@@ -1,6 +1,8 @@
#ifndef JEMALLOC_INTERNAL_PSSET_H
#define JEMALLOC_INTERNAL_PSSET_H
+#include "jemalloc/internal/hpdata.h"
+
/*
* A page-slab set. What the eset is to PAC, the psset is to HPA. It maintains
* a collection of page-slabs (the intent being that they are backed by
@@ -51,21 +53,18 @@ struct psset_s {
* The pageslabs, quantized by the size class of the largest contiguous
* free run of pages in a pageslab.
*/
- edata_age_heap_t pageslabs[PSSET_NPSIZES];
+ hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
psset_stats_t stats;
-
- /* How many alloc_new calls have happened? */
- uint64_t age_counter;
};
void psset_init(psset_t *psset);
void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src);
-void psset_insert(psset_t *psset, edata_t *ps);
-void psset_remove(psset_t *psset, edata_t *ps);
+void psset_insert(psset_t *psset, hpdata_t *ps);
+void psset_remove(psset_t *psset, hpdata_t *ps);
-void psset_hugify(psset_t *psset, edata_t *ps);
+void psset_hugify(psset_t *psset, hpdata_t *ps);
/*
* Tries to obtain a chunk from an existing pageslab already in the set.
@@ -78,7 +77,7 @@ bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
* to the psset and allocate an extent from within it. The passed-in pageslab
* must be at least as big as size.
*/
-void psset_alloc_new(psset_t *psset, edata_t *ps,
+void psset_alloc_new(psset_t *psset, hpdata_t *ps,
edata_t *r_edata, size_t size);
/*
@@ -89,6 +88,6 @@ void psset_alloc_new(psset_t *psset, edata_t *ps,
* result must be checked and deallocated to the central HPA. Otherwise returns
* NULL.
*/
-edata_t *psset_dalloc(psset_t *psset, edata_t *edata);
+hpdata_t *psset_dalloc(psset_t *psset, edata_t *edata);
#endif /* JEMALLOC_INTERNAL_PSSET_H */
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index 6c4e7fdc..531dd9a6 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -62,6 +62,7 @@
+
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
index 84ff5748..f031fb10 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
@@ -70,6 +70,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index 07fbe21e..bc64de5c 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -62,6 +62,7 @@
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
index 84ff5748..f031fb10 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
@@ -70,6 +70,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/src/edata.c b/src/edata.c
index a6597312..23523dd0 100644
--- a/src/edata.c
+++ b/src/edata.c
@@ -4,4 +4,3 @@
ph_gen(, edata_avail_, edata_avail_t, edata_t, ph_link,
edata_esnead_comp)
ph_gen(, edata_heap_, edata_heap_t, edata_t, ph_link, edata_snad_comp)
-ph_gen(, edata_age_heap_, edata_age_heap_t, edata_t, ph_link, edata_age_comp)
diff --git a/src/hpa.c b/src/hpa.c
index ca75628c..9a190c8a 100644
--- a/src/hpa.c
+++ b/src/hpa.c
@@ -33,22 +33,22 @@ hpa_supported() {
* We fundamentally rely on a address-space-hungry growth strategy for
* hugepages.
*/
- if (LG_SIZEOF_PTR == 2) {
+ if (LG_SIZEOF_PTR != 3) {
return false;
}
/*
- * We use the edata bitmap; it needs to have at least as many bits as a
- * hugepage has pages.
+ * If we couldn't detect the value of HUGEPAGE, HUGEPAGE_PAGES becomes
+ * this sentinel value -- see the comment in pages.h.
*/
- if (HUGEPAGE / PAGE > BITMAP_GROUPS_MAX * sizeof(bitmap_t) * 8) {
+ if (HUGEPAGE_PAGES == 1) {
return false;
}
return true;
}
bool
-hpa_shard_init(hpa_shard_t *shard, emap_t *emap, edata_cache_t *edata_cache,
- unsigned ind, size_t alloc_max) {
+hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
+ edata_cache_t *edata_cache, unsigned ind, size_t alloc_max) {
/* malloc_conf processing should have filtered out these cases. */
assert(hpa_supported());
bool err;
@@ -64,11 +64,14 @@ hpa_shard_init(hpa_shard_t *shard, emap_t *emap, edata_cache_t *edata_cache,
}
assert(edata_cache != NULL);
+ shard->base = base;
edata_cache_small_init(&shard->ecs, edata_cache);
psset_init(&shard->psset);
shard->alloc_max = alloc_max;
- edata_list_inactive_init(&shard->unused_slabs);
+ hpdata_list_init(&shard->unused_slabs);
+ shard->age_counter = 0;
shard->eden = NULL;
+ shard->eden_len = 0;
shard->ind = ind;
shard->emap = emap;
@@ -104,22 +107,27 @@ hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
malloc_mutex_unlock(tsdn, &shard->mtx);
}
+static hpdata_t *
+hpa_alloc_ps(tsdn_t *tsdn, hpa_shard_t *shard) {
+ return (hpdata_t *)base_alloc(tsdn, shard->base, sizeof(hpdata_t),
+ CACHELINE);
+}
+
static bool
-hpa_should_hugify(hpa_shard_t *shard, edata_t *ps) {
+hpa_should_hugify(hpa_shard_t *shard, hpdata_t *ps) {
/*
* For now, just use a static check; hugify a page if it's <= 5%
* inactive. Eventually, this should be a malloc conf option.
*/
- return !edata_hugeified_get(ps)
- && edata_nfree_get(ps) < (HUGEPAGE / PAGE) * 5 / 100;
+ return !hpdata_huge_get(ps)
+ && hpdata_nfree_get(ps) < (HUGEPAGE / PAGE) * 5 / 100;
}
/* Returns true on error. */
static void
-hpa_hugify(edata_t *ps) {
- assert(edata_size_get(ps) == HUGEPAGE);
- assert(edata_hugeified_get(ps));
- bool err = pages_huge(edata_base_get(ps), HUGEPAGE);
+hpa_hugify(hpdata_t *ps) {
+ assert(hpdata_huge_get(ps));
+ bool err = pages_huge(hpdata_addr_get(ps), HUGEPAGE);
/*
* Eat the error; even if the hugeification failed, it's still safe to
* pretend it didn't (and would require extraordinary measures to
@@ -129,30 +137,36 @@ hpa_hugify(edata_t *ps) {
}
static void
-hpa_dehugify(edata_t *ps) {
+hpa_dehugify(hpdata_t *ps) {
/* Purge, then dehugify while unbacked. */
- pages_purge_forced(edata_addr_get(ps), HUGEPAGE);
- pages_nohuge(edata_addr_get(ps), HUGEPAGE);
- edata_hugeified_set(ps, false);
+ pages_purge_forced(hpdata_addr_get(ps), HUGEPAGE);
+ pages_nohuge(hpdata_addr_get(ps), HUGEPAGE);
+ hpdata_huge_set(ps, false);
}
-static edata_t *
+static hpdata_t *
hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->grow_mtx);
- edata_t *ps = NULL;
+ hpdata_t *ps = NULL;
/* Is there address space waiting for reuse? */
malloc_mutex_assert_owner(tsdn, &shard->grow_mtx);
- ps = edata_list_inactive_first(&shard->unused_slabs);
+ ps = hpdata_list_first(&shard->unused_slabs);
if (ps != NULL) {
- edata_list_inactive_remove(&shard->unused_slabs, ps);
+ hpdata_list_remove(&shard->unused_slabs, ps);
+ hpdata_age_set(ps, shard->age_counter++);
return ps;
}
/* Is eden a perfect fit? */
- if (shard->eden != NULL && edata_size_get(shard->eden) == HUGEPAGE) {
- ps = shard->eden;
+ if (shard->eden != NULL && shard->eden_len == HUGEPAGE) {
+ ps = hpa_alloc_ps(tsdn, shard);
+ if (ps == NULL) {
+ return NULL;
+ }
+ hpdata_init(ps, shard->eden, shard->age_counter++);
shard->eden = NULL;
+ shard->eden_len = 0;
return ps;
}
@@ -173,78 +187,32 @@ hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) {
if (new_eden == NULL) {
return NULL;
}
- malloc_mutex_lock(tsdn, &shard->mtx);
- /* Allocate ps edata, bailing if we fail. */
- ps = edata_cache_small_get(tsdn, &shard->ecs);
+ ps = hpa_alloc_ps(tsdn, shard);
if (ps == NULL) {
- malloc_mutex_unlock(tsdn, &shard->mtx);
pages_unmap(new_eden, HPA_EDEN_SIZE);
return NULL;
}
- /* Allocate eden edata, bailing if we fail. */
- shard->eden = edata_cache_small_get(tsdn, &shard->ecs);
- if (shard->eden == NULL) {
- edata_cache_small_put(tsdn, &shard->ecs, ps);
- malloc_mutex_unlock(tsdn, &shard->mtx);
- pages_unmap(new_eden, HPA_EDEN_SIZE);
- return NULL;
- }
- /* Success. */
- malloc_mutex_unlock(tsdn, &shard->mtx);
-
- /*
- * Note that the values here don't really make sense (e.g. eden
- * is actually zeroed). But we don't use the slab metadata in
- * determining subsequent allocation metadata (e.g. zero
- * tracking should be done at the per-page level, not at the
- * level of the hugepage). It's just a convenient data
- * structure that contains much of the helpers we need (defined
- * lists, a bitmap, an address field, etc.). Eventually, we'll
- * have a "real" representation of a hugepage that's unconnected
- * to the edata_ts it will serve allocations into.
- */
- edata_init(shard->eden, shard->ind, new_eden, HPA_EDEN_SIZE,
- /* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_dirty,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- /* is_head */ true);
- edata_hugeified_set(shard->eden, false);
+ shard->eden = new_eden;
+ shard->eden_len = HPA_EDEN_SIZE;
} else {
/* Eden is already nonempty; only need an edata for ps. */
- malloc_mutex_lock(tsdn, &shard->mtx);
- ps = edata_cache_small_get(tsdn, &shard->ecs);
- malloc_mutex_unlock(tsdn, &shard->mtx);
+ ps = hpa_alloc_ps(tsdn, shard);
if (ps == NULL) {
return NULL;
}
}
- /*
- * We should have dropped mtx since we're not touching ecs any more, but
- * we should continue to hold the grow mutex, since we're about to touch
- * eden.
- */
- malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
- malloc_mutex_assert_owner(tsdn, &shard->grow_mtx);
-
+ assert(ps != NULL);
assert(shard->eden != NULL);
- assert(edata_size_get(shard->eden) > HUGEPAGE);
- assert(edata_size_get(shard->eden) % HUGEPAGE == 0);
- assert(edata_addr_get(shard->eden)
- == HUGEPAGE_ADDR2BASE(edata_addr_get(shard->eden)));
- malloc_mutex_lock(tsdn, &shard->mtx);
- ps = edata_cache_small_get(tsdn, &shard->ecs);
- malloc_mutex_unlock(tsdn, &shard->mtx);
- if (ps == NULL) {
- return NULL;
- }
- edata_init(ps, edata_arena_ind_get(shard->eden),
- edata_addr_get(shard->eden), HUGEPAGE, /* slab */ false,
- /* szind */ SC_NSIZES, /* sn */ 0, extent_state_dirty,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- /* is_head */ true);
- edata_hugeified_set(ps, false);
- edata_addr_set(shard->eden, edata_past_get(ps));
- edata_size_set(shard->eden,
- edata_size_get(shard->eden) - HUGEPAGE);
+ assert(shard->eden_len > HUGEPAGE);
+ assert(shard->eden_len % HUGEPAGE == 0);
+ assert(HUGEPAGE_ADDR2BASE(shard->eden) == shard->eden);
+
+ hpdata_init(ps, shard->eden, shard->age_counter++);
+
+ char *eden_char = (char *)shard->eden;
+ eden_char += HUGEPAGE;
+ shard->eden = (void *)eden_char;
+ shard->eden_len -= HUGEPAGE;
return ps;
}
@@ -255,7 +223,7 @@ hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) {
* their address space in a list outside the psset.
*/
static void
-hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *ps) {
+hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, hpdata_t *ps) {
/*
* We do relatively expensive system calls. The ps was evicted, so no
* one should touch it while we're also touching it.
@@ -263,9 +231,6 @@ hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *ps) {
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
malloc_mutex_assert_not_owner(tsdn, &shard->grow_mtx);
- assert(edata_size_get(ps) == HUGEPAGE);
- assert(HUGEPAGE_ADDR2BASE(edata_addr_get(ps)) == edata_addr_get(ps));
-
/*
* We do this unconditionally, even for pages which were not originally
* hugeified; it has the same effect.
@@ -273,7 +238,7 @@ hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *ps) {
hpa_dehugify(ps);
malloc_mutex_lock(tsdn, &shard->grow_mtx);
- edata_list_inactive_prepend(&shard->unused_slabs, ps);
+ hpdata_list_prepend(&shard->unused_slabs, ps);
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
}
@@ -307,7 +272,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
err = emap_register_boundary(tsdn, shard->emap, edata,
SC_NSIZES, /* slab */ false);
if (err) {
- edata_t *ps = psset_dalloc(&shard->psset, edata);
+ hpdata_t *ps = psset_dalloc(&shard->psset, edata);
/*
* The pageslab was nonempty before we started; it
* should still be nonempty now, and so shouldn't get
@@ -320,7 +285,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
return NULL;
}
- edata_t *ps = edata_ps_get(edata);
+ hpdata_t *ps = edata_ps_get(edata);
assert(ps != NULL);
bool hugify = hpa_should_hugify(shard, ps);
if (hugify) {
@@ -378,16 +343,11 @@ hpa_alloc_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size) {
* deallocations (and allocations of smaller sizes) may still succeed
* while we're doing this potentially expensive system call.
*/
- edata_t *grow_edata = hpa_grow(tsdn, shard);
- if (grow_edata == NULL) {
+ hpdata_t *grow_ps = hpa_grow(tsdn, shard);
+ if (grow_ps == NULL) {
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
return NULL;
}
- assert(edata_arena_ind_get(grow_edata) == shard->ind);
-
- edata_slab_set(grow_edata, true);
- fb_group_t *fb = edata_slab_data_get(grow_edata)->bitmap;
- fb_init(fb, HUGEPAGE / PAGE);
/* We got the new edata; allocate from it. */
malloc_mutex_lock(tsdn, &shard->mtx);
@@ -395,18 +355,19 @@ hpa_alloc_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size) {
if (edata == NULL) {
malloc_mutex_unlock(tsdn, &shard->mtx);
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
+ hpa_handle_ps_eviction(tsdn, shard, grow_ps);
return NULL;
}
- psset_alloc_new(&shard->psset, grow_edata, edata, size);
+ psset_alloc_new(&shard->psset, grow_ps, edata, size);
err = emap_register_boundary(tsdn, shard->emap, edata,
SC_NSIZES, /* slab */ false);
if (err) {
- edata_t *ps = psset_dalloc(&shard->psset, edata);
+ hpdata_t *ps = psset_dalloc(&shard->psset, edata);
/*
* The pageslab was empty except for the new allocation; it
* should get evicted.
*/
- assert(ps == grow_edata);
+ assert(ps == grow_ps);
edata_cache_small_put(tsdn, &shard->ecs, edata);
/*
* Technically the same as fallthrough at the time of this
@@ -496,7 +457,7 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
assert(edata_committed_get(edata));
assert(edata_base_get(edata) != NULL);
- edata_t *ps = edata_ps_get(edata);
+ hpdata_t *ps = edata_ps_get(edata);
/* Currently, all edatas come from pageslabs. */
assert(ps != NULL);
emap_deregister_boundary(tsdn, shard->emap, edata);
@@ -506,7 +467,7 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
* Page slabs can move between pssets (and have their hugeified status
* change) in racy ways.
*/
- edata_t *evicted_ps = psset_dalloc(&shard->psset, edata);
+ hpdata_t *evicted_ps = psset_dalloc(&shard->psset, edata);
/*
* If a pageslab became empty because of the dalloc, it better have been
* the one we expected.
@@ -562,11 +523,10 @@ hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_assert_empty(tsdn, shard, &shard->psset);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
- edata_t *ps;
- while ((ps = edata_list_inactive_first(&shard->unused_slabs)) != NULL) {
- assert(edata_size_get(ps) == HUGEPAGE);
- edata_list_inactive_remove(&shard->unused_slabs, ps);
- pages_unmap(edata_base_get(ps), HUGEPAGE);
+ hpdata_t *ps;
+ while ((ps = hpdata_list_first(&shard->unused_slabs)) != NULL) {
+ hpdata_list_remove(&shard->unused_slabs, ps);
+ pages_unmap(hpdata_addr_get(ps), HUGEPAGE);
}
}
diff --git a/src/hpdata.c b/src/hpdata.c
new file mode 100644
index 00000000..bbe3acce
--- /dev/null
+++ b/src/hpdata.c
@@ -0,0 +1,18 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/hpdata.h"
+
+static int
+hpdata_age_comp(const hpdata_t *a, const hpdata_t *b) {
+ uint64_t a_age = hpdata_age_get(a);
+ uint64_t b_age = hpdata_age_get(b);
+ /*
+ * hpdata ages are operation counts in the psset; no two should be the
+ * same.
+ */
+ assert(a_age != b_age);
+ return (a_age > b_age) - (a_age < b_age);
+}
+
+ph_gen(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t, ph_link, hpdata_age_comp)
diff --git a/src/pa.c b/src/pa.c
index bc52ff43..da64b829 100644
--- a/src/pa.c
+++ b/src/pa.c
@@ -51,8 +51,8 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
bool
pa_shard_enable_hpa(pa_shard_t *shard, size_t alloc_max, size_t sec_nshards,
size_t sec_alloc_max, size_t sec_bytes_max) {
- if (hpa_shard_init(&shard->hpa_shard, shard->emap, &shard->edata_cache,
- shard->ind, alloc_max)) {
+ if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base,
+ &shard->edata_cache, shard->ind, alloc_max)) {
return true;
}
if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, sec_nshards,
diff --git a/src/psset.c b/src/psset.c
index 2ee683b6..cebc1ce8 100644
--- a/src/psset.c
+++ b/src/psset.c
@@ -11,11 +11,10 @@ static const bitmap_info_t psset_bitmap_info =
void
psset_init(psset_t *psset) {
for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
- edata_age_heap_new(&psset->pageslabs[i]);
+ hpdata_age_heap_new(&psset->pageslabs[i]);
}
bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
memset(&psset->stats, 0, sizeof(psset->stats));
- psset->age_counter = 0;
}
static void
@@ -49,18 +48,17 @@ psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) {
* ensure we don't miss any heap modification operations.
*/
JEMALLOC_ALWAYS_INLINE void
-psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, edata_t *ps,
+psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps,
bool insert) {
- size_t *npageslabs_dst = edata_hugeified_get(ps)
+ size_t *npageslabs_dst = hpdata_huge_get(ps)
? &binstats->npageslabs_huge : &binstats->npageslabs_nonhuge;
- size_t *nactive_dst = edata_hugeified_get(ps)
+ size_t *nactive_dst = hpdata_huge_get(ps)
? &binstats->nactive_huge : &binstats->nactive_nonhuge;
- size_t *ninactive_dst = edata_hugeified_get(ps)
+ size_t *ninactive_dst = hpdata_huge_get(ps)
? &binstats->ninactive_huge : &binstats->ninactive_nonhuge;
- size_t npages = edata_size_get(ps) >> LG_PAGE;
- size_t ninactive = edata_nfree_get(ps);
- size_t nactive = npages - ninactive;
+ size_t ninactive = hpdata_nfree_get(ps);
+ size_t nactive = HUGEPAGE_PAGES - ninactive;
size_t mul = insert ? (size_t)1 : (size_t)-1;
*npageslabs_dst += mul * 1;
@@ -69,12 +67,12 @@ psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, edata_t *ps,
}
static void
-psset_bin_stats_insert(psset_bin_stats_t *binstats, edata_t *ps) {
+psset_bin_stats_insert(psset_bin_stats_t *binstats, hpdata_t *ps) {
psset_bin_stats_insert_remove(binstats, ps, /* insert */ true);
}
static void
-psset_bin_stats_remove(psset_bin_stats_t *binstats, edata_t *ps) {
+psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) {
psset_bin_stats_insert_remove(binstats, ps, /* insert */ false);
}
@@ -96,27 +94,27 @@ psset_bin_stats_deactivate(psset_bin_stats_t *binstats, bool huge, size_t num) {
}
static void
-psset_edata_heap_remove(psset_t *psset, pszind_t pind, edata_t *ps) {
- edata_age_heap_remove(&psset->pageslabs[pind], ps);
+psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
+ hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
psset_bin_stats_remove(&psset->stats.nonfull_slabs[pind], ps);
}
static void
-psset_edata_heap_insert(psset_t *psset, pszind_t pind, edata_t *ps) {
- edata_age_heap_insert(&psset->pageslabs[pind], ps);
+psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
+ hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
psset_bin_stats_insert(&psset->stats.nonfull_slabs[pind], ps);
}
JEMALLOC_ALWAYS_INLINE void
-psset_assert_ps_consistent(edata_t *ps) {
- assert(fb_urange_longest(edata_slab_data_get(ps)->bitmap,
- edata_size_get(ps) >> LG_PAGE) == edata_longest_free_range_get(ps));
+psset_assert_ps_consistent(hpdata_t *ps) {
+ assert(fb_urange_longest(ps->active_pages, HUGEPAGE_PAGES)
+ == hpdata_longest_free_range_get(ps));
}
void
-psset_insert(psset_t *psset, edata_t *ps) {
+psset_insert(psset_t *psset, hpdata_t *ps) {
psset_assert_ps_consistent(ps);
- size_t longest_free_range = edata_longest_free_range_get(ps);
+ size_t longest_free_range = hpdata_longest_free_range_get(ps);
if (longest_free_range == 0) {
/*
@@ -131,16 +129,16 @@ psset_insert(psset_t *psset, edata_t *ps) {
longest_free_range << LG_PAGE));
assert(pind < PSSET_NPSIZES);
- if (edata_age_heap_empty(&psset->pageslabs[pind])) {
+ if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
}
- psset_edata_heap_insert(psset, pind, ps);
+ psset_hpdata_heap_insert(psset, pind, ps);
}
void
-psset_remove(psset_t *psset, edata_t *ps) {
+psset_remove(psset_t *psset, hpdata_t *ps) {
psset_assert_ps_consistent(ps);
- size_t longest_free_range = edata_longest_free_range_get(ps);
+ size_t longest_free_range = hpdata_longest_free_range_get(ps);
if (longest_free_range == 0) {
psset_bin_stats_remove(&psset->stats.full_slabs, ps);
@@ -150,18 +148,18 @@ psset_remove(psset_t *psset, edata_t *ps) {
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
longest_free_range << LG_PAGE));
assert(pind < PSSET_NPSIZES);
- psset_edata_heap_remove(psset, pind, ps);
- if (edata_age_heap_empty(&psset->pageslabs[pind])) {
+ psset_hpdata_heap_remove(psset, pind, ps);
+ if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind);
}
}
void
-psset_hugify(psset_t *psset, edata_t *ps) {
- assert(!edata_hugeified_get(ps));
+psset_hugify(psset_t *psset, hpdata_t *ps) {
+ assert(!hpdata_huge_get(ps));
psset_assert_ps_consistent(ps);
- size_t longest_free_range = edata_longest_free_range_get(ps);
+ size_t longest_free_range = hpdata_longest_free_range_get(ps);
psset_bin_stats_t *bin_stats;
if (longest_free_range == 0) {
bin_stats = &psset->stats.full_slabs;
@@ -172,7 +170,7 @@ psset_hugify(psset_t *psset, edata_t *ps) {
bin_stats = &psset->stats.nonfull_slabs[pind];
}
psset_bin_stats_remove(bin_stats, ps);
- edata_hugeified_set(ps, true);
+ hpdata_huge_set(ps, true);
psset_bin_stats_insert(bin_stats, ps);
}
@@ -180,7 +178,7 @@ psset_hugify(psset_t *psset, edata_t *ps) {
* Similar to PAC's extent_recycle_extract. Out of all the pageslabs in the
* set, picks one that can satisfy the allocation and remove it from the set.
*/
-static edata_t *
+static hpdata_t *
psset_recycle_extract(psset_t *psset, size_t size) {
pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
@@ -188,13 +186,13 @@ psset_recycle_extract(psset_t *psset, size_t size) {
if (pind == PSSET_NPSIZES) {
return NULL;
}
- edata_t *ps = edata_age_heap_first(&psset->pageslabs[pind]);
+ hpdata_t *ps = hpdata_age_heap_first(&psset->pageslabs[pind]);
if (ps == NULL) {
return NULL;
}
- psset_edata_heap_remove(psset, pind, ps);
- if (edata_age_heap_empty(&psset->pageslabs[pind])) {
+ psset_hpdata_heap_remove(psset, pind, ps);
+ if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info, pind);
}
@@ -207,7 +205,7 @@ psset_recycle_extract(psset_t *psset, size_t size) {
* edata with a range in the pageslab, and puts ps back in the set.
*/
static void
-psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
+psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
size_t size) {
size_t start = 0;
/*
@@ -217,15 +215,14 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
size_t begin = 0;
size_t len = 0;
- fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
+ fb_group_t *ps_fb = ps->active_pages;
size_t npages = size >> LG_PAGE;
- size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
size_t largest_unchosen_range = 0;
while (true) {
- bool found = fb_urange_iter(ps_fb, ps_npages, start, &begin,
- &len);
+ bool found = fb_urange_iter(ps_fb, HUGEPAGE_PAGES, start,
+ &begin, &len);
/*
* A precondition to this function is that ps must be able to
* serve the allocation.
@@ -245,14 +242,14 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
}
start = begin + len;
}
- uintptr_t addr = (uintptr_t)edata_base_get(ps) + begin * PAGE;
+ uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size,
/* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active,
/* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
EXTENT_NOT_HEAD);
edata_ps_set(r_edata, ps);
- fb_set_range(ps_fb, ps_npages, begin, npages);
- edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) - npages));
+ fb_set_range(ps_fb, HUGEPAGE_PAGES, begin, npages);
+ hpdata_nfree_set(ps, (uint32_t)(hpdata_nfree_get(ps) - npages));
/* The pageslab isn't in a bin, so no bin stats need to change. */
/*
@@ -267,8 +264,8 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
* this check in the case where we're allocating from some smaller run.
*/
start = begin + npages;
- while (start < ps_npages) {
- bool found = fb_urange_iter(ps_fb, ps_npages, start, &begin,
+ while (start < HUGEPAGE_PAGES) {
+ bool found = fb_urange_iter(ps_fb, HUGEPAGE_PAGES, start, &begin,
&len);
if (!found) {
break;
@@ -278,7 +275,7 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
}
start = begin + len;
}
- edata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range);
+ hpdata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range);
if (largest_unchosen_range == 0) {
psset_bin_stats_insert(&psset->stats.full_slabs, ps);
} else {
@@ -288,7 +285,7 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
bool
psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) {
- edata_t *ps = psset_recycle_extract(psset, size);
+ hpdata_t *ps = psset_recycle_extract(psset, size);
if (ps == NULL) {
return true;
}
@@ -297,48 +294,43 @@ psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) {
}
void
-psset_alloc_new(psset_t *psset, edata_t *ps, edata_t *r_edata, size_t size) {
- fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
- size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
- assert(fb_empty(ps_fb, ps_npages));
- assert(ps_npages >= (size >> LG_PAGE));
- edata_nfree_set(ps, (uint32_t)ps_npages);
- edata_age_set(ps, psset->age_counter);
- psset->age_counter++;
+psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
+ fb_group_t *ps_fb = ps->active_pages;
+ assert(fb_empty(ps_fb, HUGEPAGE_PAGES));
+ assert(hpdata_nfree_get(ps) == HUGEPAGE_PAGES);
psset_ps_alloc_insert(psset, ps, r_edata, size);
}
-edata_t *
+hpdata_t *
psset_dalloc(psset_t *psset, edata_t *edata) {
assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
assert(edata_ps_get(edata) != NULL);
- edata_t *ps = edata_ps_get(edata);
+ hpdata_t *ps = edata_ps_get(edata);
- fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
- size_t ps_old_longest_free_range = edata_longest_free_range_get(ps);
+ fb_group_t *ps_fb = ps->active_pages;
+ size_t ps_old_longest_free_range = hpdata_longest_free_range_get(ps);
pszind_t old_pind = SC_NPSIZES;
if (ps_old_longest_free_range != 0) {
old_pind = sz_psz2ind(sz_psz_quantize_floor(
ps_old_longest_free_range << LG_PAGE));
}
- size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
size_t begin =
- ((uintptr_t)edata_base_get(edata) - (uintptr_t)edata_base_get(ps))
+ ((uintptr_t)edata_base_get(edata) - (uintptr_t)hpdata_addr_get(ps))
>> LG_PAGE;
size_t len = edata_size_get(edata) >> LG_PAGE;
- fb_unset_range(ps_fb, ps_npages, begin, len);
+ fb_unset_range(ps_fb, HUGEPAGE_PAGES, begin, len);
/* The pageslab is still in the bin; adjust its stats first. */
psset_bin_stats_t *bin_stats = (ps_old_longest_free_range == 0
? &psset->stats.full_slabs : &psset->stats.nonfull_slabs[old_pind]);
- psset_bin_stats_deactivate(bin_stats, edata_hugeified_get(ps), len);
+ psset_bin_stats_deactivate(bin_stats, hpdata_huge_get(ps), len);
- edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) + len));
+ hpdata_nfree_set(ps, (uint32_t)(hpdata_nfree_get(ps) + len));
/* We might have just created a new, larger range. */
- size_t new_begin = (size_t)(fb_fls(ps_fb, ps_npages, begin) + 1);
- size_t new_end = fb_ffs(ps_fb, ps_npages, begin + len - 1);
+ size_t new_begin = (size_t)(fb_fls(ps_fb, HUGEPAGE_PAGES, begin) + 1);
+ size_t new_end = fb_ffs(ps_fb, HUGEPAGE_PAGES, begin + len - 1);
size_t new_range_len = new_end - new_begin;
/*
* If the new free range is no longer than the previous longest one,
@@ -352,7 +344,7 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
* Otherwise, it might need to get evicted from the set, or change its
* bin.
*/
- edata_longest_free_range_set(ps, (uint32_t)new_range_len);
+ hpdata_longest_free_range_set(ps, (uint32_t)new_range_len);
/*
* If it was previously non-full, then it's in some (possibly now
* incorrect) bin already; remove it.
@@ -366,8 +358,8 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
* and the issue becomes moot).
*/
if (ps_old_longest_free_range > 0) {
- psset_edata_heap_remove(psset, old_pind, ps);
- if (edata_age_heap_empty(&psset->pageslabs[old_pind])) {
+ psset_hpdata_heap_remove(psset, old_pind, ps);
+ if (hpdata_age_heap_empty(&psset->pageslabs[old_pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info,
(size_t)old_pind);
}
@@ -379,16 +371,16 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
psset_bin_stats_remove(&psset->stats.full_slabs, ps);
}
/* If the pageslab is empty, it gets evicted from the set. */
- if (new_range_len == ps_npages) {
+ if (new_range_len == HUGEPAGE_PAGES) {
return ps;
}
/* Otherwise, it gets reinserted. */
pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor(
new_range_len << LG_PAGE));
- if (edata_age_heap_empty(&psset->pageslabs[new_pind])) {
+ if (hpdata_age_heap_empty(&psset->pageslabs[new_pind])) {
bitmap_unset(psset->bitmap, &psset_bitmap_info,
(size_t)new_pind);
}
- psset_edata_heap_insert(psset, new_pind, ps);
+ psset_hpdata_heap_insert(psset, new_pind, ps);
return NULL;
}
diff --git a/test/unit/hpa.c b/test/unit/hpa.c
index 94efd4ae..90ec89e4 100644
--- a/test/unit/hpa.c
+++ b/test/unit/hpa.c
@@ -38,7 +38,8 @@ create_test_data() {
assert_false(err, "");
err = hpa_shard_init(&test_data->shard, &test_data->emap,
- &test_data->shard_edata_cache, SHARD_IND, ALLOC_MAX);
+ test_data->base, &test_data->shard_edata_cache, SHARD_IND,
+ ALLOC_MAX);
assert_false(err, "");
return (hpa_shard_t *)test_data;
diff --git a/test/unit/psset.c b/test/unit/psset.c
index ea61ab92..811c7be1 100644
--- a/test/unit/psset.c
+++ b/test/unit/psset.c
@@ -2,10 +2,8 @@
#include "jemalloc/internal/psset.h"
-#define PAGESLAB_PAGES (HUGEPAGE / PAGE)
-#define PAGESLAB_SIZE (PAGESLAB_PAGES << LG_PAGE)
-#define PAGESLAB_SN 123
-#define PAGESLAB_ADDR ((void *)(1234 << LG_PAGE))
+#define PAGESLAB_ADDR ((void *)(1234 * HUGEPAGE))
+#define PAGESLAB_AGE 5678
#define ALLOC_ARENA_IND 111
#define ALLOC_ESN 222
@@ -42,14 +40,10 @@ edata_expect(edata_t *edata, size_t page_offset, size_t page_cnt) {
TEST_BEGIN(test_empty) {
bool err;
- edata_t pageslab;
- memset(&pageslab, 0, sizeof(pageslab));
- edata_t alloc;
+ hpdata_t pageslab;
+ hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE);
- edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
- /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- EXTENT_IS_HEAD);
+ edata_t alloc;
edata_init_test(&alloc);
psset_t psset;
@@ -63,27 +57,24 @@ TEST_END
TEST_BEGIN(test_fill) {
bool err;
- edata_t pageslab;
- memset(&pageslab, 0, sizeof(pageslab));
- edata_t alloc[PAGESLAB_PAGES];
- edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
- /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- EXTENT_IS_HEAD);
+ hpdata_t pageslab;
+ hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE);
+
+ edata_t alloc[HUGEPAGE_PAGES];
psset_t psset;
psset_init(&psset);
edata_init_test(&alloc[0]);
psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
- for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 1; i < HUGEPAGE_PAGES; i++) {
edata_init_test(&alloc[i]);
err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
expect_false(err, "Nonempty psset failed page allocation.");
}
- for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 0; i < HUGEPAGE_PAGES; i++) {
edata_t *edata = &alloc[i];
edata_expect(edata, i, 1);
}
@@ -98,30 +89,26 @@ TEST_END
TEST_BEGIN(test_reuse) {
bool err;
- edata_t *ps;
+ hpdata_t *ps;
- edata_t pageslab;
- memset(&pageslab, 0, sizeof(pageslab));
- edata_t alloc[PAGESLAB_PAGES];
+ hpdata_t pageslab;
+ hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE);
- edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
- /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- EXTENT_IS_HEAD);
+ edata_t alloc[HUGEPAGE_PAGES];
psset_t psset;
psset_init(&psset);
edata_init_test(&alloc[0]);
psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
- for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 1; i < HUGEPAGE_PAGES; i++) {
edata_init_test(&alloc[i]);
err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
expect_false(err, "Nonempty psset failed page allocation.");
}
/* Free odd indices. */
- for (size_t i = 0; i < PAGESLAB_PAGES; i ++) {
+ for (size_t i = 0; i < HUGEPAGE_PAGES; i ++) {
if (i % 2 == 0) {
continue;
}
@@ -129,7 +116,7 @@ TEST_BEGIN(test_reuse) {
expect_ptr_null(ps, "Nonempty pageslab evicted");
}
/* Realloc into them. */
- for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 0; i < HUGEPAGE_PAGES; i++) {
if (i % 2 == 0) {
continue;
}
@@ -138,7 +125,7 @@ TEST_BEGIN(test_reuse) {
edata_expect(&alloc[i], i, 1);
}
/* Now, free the pages at indices 0 or 1 mod 2. */
- for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 0; i < HUGEPAGE_PAGES; i++) {
if (i % 4 > 1) {
continue;
}
@@ -146,7 +133,7 @@ TEST_BEGIN(test_reuse) {
expect_ptr_null(ps, "Nonempty pageslab evicted");
}
/* And realloc 2-page allocations into them. */
- for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 0; i < HUGEPAGE_PAGES; i++) {
if (i % 4 != 0) {
continue;
}
@@ -155,7 +142,7 @@ TEST_BEGIN(test_reuse) {
edata_expect(&alloc[i], i, 2);
}
/* Free all the 2-page allocations. */
- for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 0; i < HUGEPAGE_PAGES; i++) {
if (i % 4 != 0) {
continue;
}
@@ -175,13 +162,13 @@ TEST_BEGIN(test_reuse) {
edata_expect(&alloc[index_of_3], index_of_3, 3);
/* Free up a 4-page hole at the end. */
- ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]);
+ ps = psset_dalloc(&psset, &alloc[HUGEPAGE_PAGES - 1]);
expect_ptr_null(ps, "Nonempty pageslab evicted");
- ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 2]);
+ ps = psset_dalloc(&psset, &alloc[HUGEPAGE_PAGES - 2]);
expect_ptr_null(ps, "Nonempty pageslab evicted");
/* Make sure we can satisfy an allocation at the very end of a slab. */
- size_t index_of_4 = PAGESLAB_PAGES - 4;
+ size_t index_of_4 = HUGEPAGE_PAGES - 4;
ps = psset_dalloc(&psset, &alloc[index_of_4]);
expect_ptr_null(ps, "Nonempty pageslab evicted");
err = psset_alloc_reuse(&psset, &alloc[index_of_4], 4 * PAGE);
@@ -192,33 +179,31 @@ TEST_END
TEST_BEGIN(test_evict) {
bool err;
- edata_t *ps;
- edata_t pageslab;
- memset(&pageslab, 0, sizeof(pageslab));
- edata_t alloc[PAGESLAB_PAGES];
+ hpdata_t *ps;
+
+ hpdata_t pageslab;
+ hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE);
+
+ edata_t alloc[HUGEPAGE_PAGES];
- edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
- /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- EXTENT_IS_HEAD);
psset_t psset;
psset_init(&psset);
/* Alloc the whole slab. */
edata_init_test(&alloc[0]);
psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
- for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 1; i < HUGEPAGE_PAGES; i++) {
edata_init_test(&alloc[i]);
err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
expect_false(err, "Unxpected allocation failure");
}
/* Dealloc the whole slab, going forwards. */
- for (size_t i = 0; i < PAGESLAB_PAGES - 1; i++) {
+ for (size_t i = 0; i < HUGEPAGE_PAGES - 1; i++) {
ps = psset_dalloc(&psset, &alloc[i]);
expect_ptr_null(ps, "Nonempty pageslab evicted");
}
- ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]);
+ ps = psset_dalloc(&psset, &alloc[HUGEPAGE_PAGES - 1]);
expect_ptr_eq(&pageslab, ps, "Empty pageslab not evicted.");
err = psset_alloc_reuse(&psset, &alloc[0], PAGE);
@@ -228,20 +213,15 @@ TEST_END
TEST_BEGIN(test_multi_pageslab) {
bool err;
- edata_t *ps;
- edata_t pageslab[2];
- memset(&pageslab, 0, sizeof(pageslab));
- edata_t alloc[2][PAGESLAB_PAGES];
+ hpdata_t *ps;
- edata_init(&pageslab[0], /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
- /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- EXTENT_IS_HEAD);
- edata_init(&pageslab[1], /* arena_ind */ 0,
- (void *)((uintptr_t)PAGESLAB_ADDR + PAGESLAB_SIZE), PAGESLAB_SIZE,
- /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- EXTENT_IS_HEAD);
+ hpdata_t pageslab[2];
+ hpdata_init(&pageslab[0], PAGESLAB_ADDR, PAGESLAB_AGE);
+ hpdata_init(&pageslab[1],
+ (void *)((uintptr_t)PAGESLAB_ADDR + HUGEPAGE),
+ PAGESLAB_AGE + 1);
+
+ edata_t alloc[2][HUGEPAGE_PAGES];
psset_t psset;
psset_init(&psset);
@@ -254,7 +234,7 @@ TEST_BEGIN(test_multi_pageslab) {
/* Fill them both up; make sure we do so in first-fit order. */
for (size_t i = 0; i < 2; i++) {
- for (size_t j = 1; j < PAGESLAB_PAGES; j++) {
+ for (size_t j = 1; j < HUGEPAGE_PAGES; j++) {
edata_init_test(&alloc[i][j]);
err = psset_alloc_reuse(&psset, &alloc[i][j], PAGE);
expect_false(err,
@@ -306,10 +286,10 @@ stats_expect_empty(psset_bin_stats_t *stats) {
static void
stats_expect(psset_t *psset, size_t nactive) {
- if (nactive == PAGESLAB_PAGES) {
+ if (nactive == HUGEPAGE_PAGES) {
expect_zu_eq(1, psset->stats.full_slabs.npageslabs_nonhuge,
"Expected a full slab");
- expect_zu_eq(PAGESLAB_PAGES,
+ expect_zu_eq(HUGEPAGE_PAGES,
psset->stats.full_slabs.nactive_nonhuge,
"Should have exactly filled the bin");
expect_zu_eq(0, psset->stats.full_slabs.ninactive_nonhuge,
@@ -317,9 +297,9 @@ stats_expect(psset_t *psset, size_t nactive) {
} else {
stats_expect_empty(&psset->stats.full_slabs);
}
- size_t ninactive = PAGESLAB_PAGES - nactive;
+ size_t ninactive = HUGEPAGE_PAGES - nactive;
pszind_t nonempty_pind = PSSET_NPSIZES;
- if (ninactive != 0 && ninactive < PAGESLAB_PAGES) {
+ if (ninactive != 0 && ninactive < HUGEPAGE_PAGES) {
nonempty_pind = sz_psz2ind(sz_psz_quantize_floor(
ninactive << LG_PAGE));
}
@@ -342,14 +322,11 @@ stats_expect(psset_t *psset, size_t nactive) {
TEST_BEGIN(test_stats) {
bool err;
- edata_t pageslab;
- memset(&pageslab, 0, sizeof(pageslab));
- edata_t alloc[PAGESLAB_PAGES];
- edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
- /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
- /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
- EXTENT_IS_HEAD);
+ hpdata_t pageslab;
+ hpdata_init(&pageslab, PAGESLAB_ADDR, PAGESLAB_AGE);
+
+ edata_t alloc[HUGEPAGE_PAGES];
psset_t psset;
psset_init(&psset);
@@ -357,15 +334,15 @@ TEST_BEGIN(test_stats) {
edata_init_test(&alloc[0]);
psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
- for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 1; i < HUGEPAGE_PAGES; i++) {
stats_expect(&psset, i);
edata_init_test(&alloc[i]);
err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
expect_false(err, "Nonempty psset failed page allocation.");
}
- stats_expect(&psset, PAGESLAB_PAGES);
- edata_t *ps;
- for (ssize_t i = PAGESLAB_PAGES - 1; i >= 0; i--) {
+ stats_expect(&psset, HUGEPAGE_PAGES);
+ hpdata_t *ps;
+ for (ssize_t i = HUGEPAGE_PAGES - 1; i >= 0; i--) {
ps = psset_dalloc(&psset, &alloc[i]);
expect_true((ps == NULL) == (i != 0),
"psset_dalloc should only evict a slab on the last free");
@@ -384,37 +361,28 @@ TEST_END
/*
* Fills in and inserts two pageslabs, with the first better than the second,
* and each fully allocated (into the allocations in allocs and worse_allocs,
- * each of which should be PAGESLAB_PAGES long).
+ * each of which should be HUGEPAGE_PAGES long).
*
* (There's nothing magic about these numbers; it's just useful to share the
* setup between the oldest fit and the insert/remove test).
*/
static void
-init_test_pageslabs(psset_t *psset, edata_t *pageslab, edata_t *worse_pageslab,
- edata_t *alloc, edata_t *worse_alloc) {
+init_test_pageslabs(psset_t *psset, hpdata_t *pageslab,
+ hpdata_t *worse_pageslab, edata_t *alloc, edata_t *worse_alloc) {
bool err;
- memset(pageslab, 0, sizeof(*pageslab));
- edata_init(pageslab, /* arena_ind */ 0, (void *)(10 * PAGESLAB_SIZE),
- PAGESLAB_SIZE, /* slab */ true, SC_NSIZES, PAGESLAB_SN + 1,
- extent_state_active, /* zeroed */ false, /* comitted */ true,
- EXTENT_PAI_HPA, EXTENT_IS_HEAD);
+ hpdata_init(pageslab, (void *)(10 * HUGEPAGE), PAGESLAB_AGE);
/*
- * This pageslab is better from an edata_comp_snad POV, but will be
- * added to the set after the previous one, and so should be less
- * preferred for allocations.
+ * This pageslab would be better from an address-first-fit POV, but
+ * better from an age POV.
*/
- memset(worse_pageslab, 0, sizeof(*worse_pageslab));
- edata_init(worse_pageslab, /* arena_ind */ 0,
- (void *)(9 * PAGESLAB_SIZE), PAGESLAB_SIZE, /* slab */ true,
- SC_NSIZES, PAGESLAB_SN - 1, extent_state_active, /* zeroed */ false,
- /* comitted */ true, EXTENT_PAI_HPA, EXTENT_IS_HEAD);
+ hpdata_init(worse_pageslab, (void *)(9 * HUGEPAGE), PAGESLAB_AGE + 1);
psset_init(psset);
edata_init_test(&alloc[0]);
psset_alloc_new(psset, pageslab, &alloc[0], PAGE);
- for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+ for (size_t i = 1; i < HUGEPAGE_PAGES; i++) {
edata_init_test(&alloc[i]);
err = psset_alloc_reuse(psset, &alloc[i], PAGE);
expect_false(err, "Nonempty psset failed page allocation.");
@@ -430,7 +398,7 @@ init_test_pageslabs(psset_t *psset, edata_t *pageslab, edata_t *worse_pageslab,
* Make the two pssets otherwise indistinguishable; all full except for
* a single page.
*/
- for (size_t i = 1; i < PAGESLAB_PAGES - 1; i++) {
+ for (size_t i = 1; i < HUGEPAGE_PAGES - 1; i++) {
edata_init_test(&worse_alloc[i]);
err = psset_alloc_reuse(psset, &alloc[i], PAGE);
expect_false(err, "Nonempty psset failed page allocation.");
@@ -439,17 +407,17 @@ init_test_pageslabs(psset_t *psset, edata_t *pageslab, edata_t *worse_pageslab,
}
/* Deallocate the last page from the older pageslab. */
- edata_t *evicted = psset_dalloc(psset, &alloc[PAGESLAB_PAGES - 1]);
+ hpdata_t *evicted = psset_dalloc(psset, &alloc[HUGEPAGE_PAGES - 1]);
expect_ptr_null(evicted, "Unexpected eviction");
}
TEST_BEGIN(test_oldest_fit) {
bool err;
- edata_t alloc[PAGESLAB_PAGES];
- edata_t worse_alloc[PAGESLAB_PAGES];
+ edata_t alloc[HUGEPAGE_PAGES];
+ edata_t worse_alloc[HUGEPAGE_PAGES];
- edata_t pageslab;
- edata_t worse_pageslab;
+ hpdata_t pageslab;
+ hpdata_t worse_pageslab;
psset_t psset;
@@ -468,12 +436,12 @@ TEST_END
TEST_BEGIN(test_insert_remove) {
bool err;
- edata_t *ps;
- edata_t alloc[PAGESLAB_PAGES];
- edata_t worse_alloc[PAGESLAB_PAGES];
+ hpdata_t *ps;
+ edata_t alloc[HUGEPAGE_PAGES];
+ edata_t worse_alloc[HUGEPAGE_PAGES];
- edata_t pageslab;
- edata_t worse_pageslab;
+ hpdata_t pageslab;
+ hpdata_t worse_pageslab;
psset_t psset;
@@ -482,31 +450,31 @@ TEST_BEGIN(test_insert_remove) {
/* Remove better; should still be able to alloc from worse. */
psset_remove(&psset, &pageslab);
- err = psset_alloc_reuse(&psset, &worse_alloc[PAGESLAB_PAGES - 1], PAGE);
+ err = psset_alloc_reuse(&psset, &worse_alloc[HUGEPAGE_PAGES - 1], PAGE);
expect_false(err, "Removal should still leave an empty page");
expect_ptr_eq(&worse_pageslab,
- edata_ps_get(&worse_alloc[PAGESLAB_PAGES - 1]),
+ edata_ps_get(&worse_alloc[HUGEPAGE_PAGES - 1]),
"Allocated out of wrong ps");
/*
* After deallocating the previous alloc and reinserting better, it
* should be preferred for future allocations.
*/
- ps = psset_dalloc(&psset, &worse_alloc[PAGESLAB_PAGES - 1]);
+ ps = psset_dalloc(&psset, &worse_alloc[HUGEPAGE_PAGES - 1]);
expect_ptr_null(ps, "Incorrect eviction of nonempty pageslab");
psset_insert(&psset, &pageslab);
- err = psset_alloc_reuse(&psset, &alloc[PAGESLAB_PAGES - 1], PAGE);
+ err = psset_alloc_reuse(&psset, &alloc[HUGEPAGE_PAGES - 1], PAGE);
expect_false(err, "psset should be nonempty");
- expect_ptr_eq(&pageslab, edata_ps_get(&alloc[PAGESLAB_PAGES - 1]),
+ expect_ptr_eq(&pageslab, edata_ps_get(&alloc[HUGEPAGE_PAGES - 1]),
"Removal/reinsertion shouldn't change ordering");
/*
* After deallocating and removing both, allocations should fail.
*/
- ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]);
+ ps = psset_dalloc(&psset, &alloc[HUGEPAGE_PAGES - 1]);
expect_ptr_null(ps, "Incorrect eviction");
psset_remove(&psset, &pageslab);
psset_remove(&psset, &worse_pageslab);
- err = psset_alloc_reuse(&psset, &alloc[PAGESLAB_PAGES - 1], PAGE);
+ err = psset_alloc_reuse(&psset, &alloc[HUGEPAGE_PAGES - 1], PAGE);
expect_true(err, "psset should be empty, but an alloc succeeded");
}
TEST_END