diff --git a/Makefile.in b/Makefile.in
index 3697e071..4769d48f 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -136,6 +136,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/prof_log.c \
$(srcroot)src/prof_recent.c \
$(srcroot)src/prof_sys.c \
+ $(srcroot)src/psset.c \
$(srcroot)src/rtree.c \
$(srcroot)src/safety_check.c \
$(srcroot)src/sc.c \
@@ -239,6 +240,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/prof_tctx.c \
$(srcroot)test/unit/prof_thread_name.c \
$(srcroot)test/unit/prof_sys_thread_name.c \
+ $(srcroot)test/unit/psset.c \
$(srcroot)test/unit/ql.c \
$(srcroot)test/unit/qr.c \
$(srcroot)test/unit/rb.c \
diff --git a/include/jemalloc/internal/edata.h b/include/jemalloc/internal/edata.h
index f1ae56a4..4fee76bf 100644
--- a/include/jemalloc/internal/edata.h
+++ b/include/jemalloc/internal/edata.h
@@ -202,7 +202,31 @@ struct edata_s {
* This keeps the size of an edata_t at exactly 128 bytes on
* architectures with 8-byte pointers and 4k pages.
*/
- void *reserved1, *reserved2;
+ void *reserved1;
+ union {
+ /*
+ * We could steal a low bit from these fields to indicate what
+ * sort of "thing" this is (a page slab, an object within a page
+ * slab, or a non-pageslab range). We don't do this yet, but it
+ * would enable some extra asserts.
+ */
+
+ /*
+ * If this edata is from an HPA, it may be part of some larger
+ * pageslab. Track it if so. Otherwise (either because it's
+ * not part of a pageslab, or not from the HPA at all), NULL.
+ */
+ edata_t *ps;
+ /*
+ * If this edata *is* a pageslab, then it has some longest free
+ * range in it. Track it.
+ */
+ struct {
+ uint32_t longest_free_range;
+ /* Not yet tracked. */
+ /* uint32_t longest_free_range_pos; */
+ };
+ };
union {
/*
@@ -346,6 +370,18 @@ edata_bsize_get(const edata_t *edata) {
return edata->e_bsize;
}
+static inline edata_t *
+edata_ps_get(const edata_t *edata) {
+ assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
+ return edata->ps;
+}
+
+static inline uint32_t
+edata_longest_free_range_get(const edata_t *edata) {
+ assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
+ return edata->longest_free_range;
+}
+
static inline void *
edata_before_get(const edata_t *edata) {
return (void *)((uintptr_t)edata_base_get(edata) - PAGE);
@@ -428,6 +464,19 @@ edata_bsize_set(edata_t *edata, size_t bsize) {
edata->e_bsize = bsize;
}
+static inline void
+edata_ps_set(edata_t *edata, edata_t *ps) {
+ assert(edata_pai_get(edata) == EXTENT_PAI_HPA || ps == NULL);
+ edata->ps = ps;
+}
+
+static inline void
+edata_longest_free_range_set(edata_t *edata, uint32_t longest_free_range) {
+ assert(edata_pai_get(edata) == EXTENT_PAI_HPA
+ || longest_free_range == 0);
+ edata->longest_free_range = longest_free_range;
+}
+
static inline void
edata_szind_set(edata_t *edata, szind_t szind) {
assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
@@ -562,6 +611,8 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
if (config_prof) {
edata_prof_tctx_set(edata, NULL);
}
+ edata_ps_set(edata, NULL);
+ edata_longest_free_range_set(edata, 0);
}
static inline void
@@ -581,6 +632,8 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, size_t sn) {
* wasting a state bit to encode this fact.
*/
edata_pai_set(edata, EXTENT_PAI_PAC);
+ edata_ps_set(edata, NULL);
+ edata_longest_free_range_set(edata, 0);
}
static inline int
diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h
new file mode 100644
index 00000000..8f3f9ee7
--- /dev/null
+++ b/include/jemalloc/internal/psset.h
@@ -0,0 +1,61 @@
+#ifndef JEMALLOC_INTERNAL_PSSET_H
+#define JEMALLOC_INTERNAL_PSSET_H
+
+/*
+ * A page-slab set. What the eset is to PAC, the psset is to HPA. It maintains
+ * a collection of page-slabs (the intent being that they are backed by
+ * hugepages, or at least could be), and handles allocation and deallocation
+ * requests.
+ *
+ * It has the same synchronization guarantees as the eset; stats queries don't
+ * need any external synchronization, everything else does.
+ */
+
+/*
+ * One more than the maximum pszind_t we will serve out of the HPA.
+ * Practically, we expect only the first few to be actually used. This
+ * corresponds to a maximum size of of 512MB on systems with 4k pages and
+ * SC_NGROUP == 4, which is already an unreasonably large maximum. Morally, you
+ * can think of this as being SC_NPSIZES, but there's no sense in wasting that
+ * much space in the arena, making bitmaps that much larger, etc.
+ */
+#define PSSET_NPSIZES 64
+
+typedef struct psset_s psset_t;
+struct psset_s {
+ /*
+ * The pageslabs, quantized by the size class of the largest contiguous
+ * free run of pages in a pageslab.
+ */
+ edata_heap_t pageslabs[PSSET_NPSIZES];
+ bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
+};
+
+void psset_init(psset_t *psset);
+
+
+/*
+ * Tries to obtain a chunk from an existing pageslab already in the set.
+ * Returns true on failure.
+ */
+bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
+
+/*
+ * Given a newly created pageslab ps (not currently in the set), pass ownership
+ * to the psset and allocate an extent from within it. The passed-in pageslab
+ * must be at least as big as size.
+ */
+void psset_alloc_new(psset_t *psset, edata_t *ps,
+ edata_t *r_edata, size_t size);
+
+/*
+ * Given an extent that comes from a pageslab in this pageslab set, returns it
+ * to its slab. Does not take ownership of the underlying edata_t.
+ *
+ * If some slab becomes empty as a result of the dalloc, it is retuend -- the
+ * result must be checked and deallocated to the central HPA. Otherwise returns
+ * NULL.
+ */
+edata_t *psset_dalloc(psset_t *psset, edata_t *edata);
+
+#endif /* JEMALLOC_INTERNAL_PSSET_H */
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index fe147790..3200eaba 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -76,6 +76,7 @@
+
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
index 4b7b6baf..8d459804 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
@@ -112,6 +112,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index 6bd43c78..7badc63c 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -76,6 +76,7 @@
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
index 4b7b6baf..8d459804 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
@@ -112,6 +112,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/src/psset.c b/src/psset.c
new file mode 100644
index 00000000..9675a0d1
--- /dev/null
+++ b/src/psset.c
@@ -0,0 +1,239 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/psset.h"
+
+#include "jemalloc/internal/flat_bitmap.h"
+
+static const bitmap_info_t psset_bitmap_info =
+ BITMAP_INFO_INITIALIZER(PSSET_NPSIZES);
+
+void
+psset_init(psset_t *psset) {
+ for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
+ edata_heap_new(&psset->pageslabs[i]);
+ }
+ bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+psset_assert_ps_consistent(edata_t *ps) {
+ assert(fb_urange_longest(edata_slab_data_get(ps)->bitmap,
+ edata_size_get(ps) >> LG_PAGE) == edata_longest_free_range_get(ps));
+}
+
+/*
+ * Similar to PAC's extent_recycle_extract. Out of all the pageslabs in the
+ * set, picks one that can satisfy the allocation and remove it from the set.
+ */
+static edata_t *
+psset_recycle_extract(psset_t *psset, size_t size) {
+ pszind_t ret_ind;
+ edata_t *ret = NULL;
+ pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(size));
+ for (pszind_t i = (pszind_t)bitmap_ffu(psset->bitmap,
+ &psset_bitmap_info, (size_t)pind);
+ i < PSSET_NPSIZES;
+ i = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
+ (size_t)i + 1)) {
+ assert(!edata_heap_empty(&psset->pageslabs[i]));
+ edata_t *ps = edata_heap_first(&psset->pageslabs[i]);
+ if (ret == NULL || edata_snad_comp(ps, ret) < 0) {
+ ret = ps;
+ ret_ind = i;
+ }
+ }
+ if (ret == NULL) {
+ return NULL;
+ }
+ edata_heap_remove(&psset->pageslabs[ret_ind], ret);
+ if (edata_heap_empty(&psset->pageslabs[ret_ind])) {
+ bitmap_set(psset->bitmap, &psset_bitmap_info, ret_ind);
+ }
+
+ psset_assert_ps_consistent(ret);
+ return ret;
+}
+
+static void
+psset_insert(psset_t *psset, edata_t *ps, size_t largest_range) {
+ psset_assert_ps_consistent(ps);
+
+ pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
+ largest_range << LG_PAGE));
+
+ assert(pind < PSSET_NPSIZES);
+
+ if (edata_heap_empty(&psset->pageslabs[pind])) {
+ bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
+ }
+ edata_heap_insert(&psset->pageslabs[pind], ps);
+}
+
+/*
+ * Given a pageslab ps and an edata to allocate size bytes from, initializes the
+ * edata with a range in the pageslab, and puts ps back in the set.
+ */
+static void
+psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
+ size_t size) {
+ size_t start = 0;
+ /*
+ * These are dead stores, but the compiler will issue warnings on them
+ * since it can't tell statically that found is always true below.
+ */
+ size_t begin = 0;
+ size_t len = 0;
+
+ fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
+
+ size_t npages = size >> LG_PAGE;
+ size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
+
+ size_t largest_unchosen_range = 0;
+ while (true) {
+ bool found = fb_urange_iter(ps_fb, ps_npages, start, &begin,
+ &len);
+ /*
+ * A precondition to this function is that ps must be able to
+ * serve the allocation.
+ */
+ assert(found);
+ if (len >= npages) {
+ /*
+ * We use first-fit within the page slabs; this gives
+ * bounded worst-case fragmentation within a slab. It's
+ * not necessarily right; we could experiment with
+ * various other options.
+ */
+ break;
+ }
+ if (len > largest_unchosen_range) {
+ largest_unchosen_range = len;
+ }
+ start = begin + len;
+ }
+ uintptr_t addr = (uintptr_t)edata_base_get(ps) + begin * PAGE;
+ edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size,
+ /* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active,
+ /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
+ EXTENT_NOT_HEAD);
+ edata_ps_set(r_edata, ps);
+ fb_set_range(ps_fb, ps_npages, begin, npages);
+ /*
+ * OK, we've got to put the pageslab back. First we have to figure out
+ * where, though; we've only checked run sizes before the pageslab we
+ * picked. We also need to look for ones after the one we picked. Note
+ * that we want begin + npages as the start position, not begin + len;
+ * we might not have used the whole range.
+ *
+ * TODO: With a little bit more care, we can guarantee that the longest
+ * free range field in the edata is accurate upon entry, and avoid doing
+ * this check in the case where we're allocating from some smaller run.
+ */
+ start = begin + npages;
+ while (start < ps_npages) {
+ bool found = fb_urange_iter(ps_fb, ps_npages, start, &begin,
+ &len);
+ if (!found) {
+ break;
+ }
+ if (len > largest_unchosen_range) {
+ largest_unchosen_range = len;
+ }
+ start = begin + len;
+ }
+ edata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range);
+ if (largest_unchosen_range != 0) {
+ psset_insert(psset, ps, largest_unchosen_range);
+ }
+}
+
+bool
+psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) {
+ edata_t *ps = psset_recycle_extract(psset, size);
+ if (ps == NULL) {
+ return true;
+ }
+ psset_ps_alloc_insert(psset, ps, r_edata, size);
+ return false;
+}
+
+void
+psset_alloc_new(psset_t *psset, edata_t *ps, edata_t *r_edata, size_t size) {
+ fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
+ size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
+ assert(fb_empty(ps_fb, ps_npages));
+
+ assert(ps_npages >= (size >> LG_PAGE));
+ psset_ps_alloc_insert(psset, ps, r_edata, size);
+}
+
+edata_t *
+psset_dalloc(psset_t *psset, edata_t *edata) {
+ assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
+ assert(edata_ps_get(edata) != NULL);
+
+ edata_t *ps = edata_ps_get(edata);
+ fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
+ size_t ps_old_longest_free_range = edata_longest_free_range_get(ps);
+
+ size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
+ size_t begin =
+ ((uintptr_t)edata_base_get(edata) - (uintptr_t)edata_base_get(ps))
+ >> LG_PAGE;
+ size_t len = edata_size_get(edata) >> LG_PAGE;
+ fb_unset_range(ps_fb, ps_npages, begin, len);
+
+ /* We might have just created a new, larger range. */
+ size_t new_begin = (size_t)(fb_fls(ps_fb, ps_npages, begin) + 1);
+ size_t new_end = fb_ffs(ps_fb, ps_npages, begin + len - 1);
+ size_t new_range_len = new_end - new_begin;
+ /*
+ * If the new free range is no longer than the previous longest one,
+ * then the pageslab is non-empty and doesn't need to change bins.
+ * We're done, and don't need to return a pageslab to evict.
+ */
+ if (new_range_len <= ps_old_longest_free_range) {
+ return NULL;
+ }
+ /*
+ * Otherwise, it might need to get evicted from the set, or change its
+ * bin.
+ */
+ edata_longest_free_range_set(ps, (uint32_t)new_range_len);
+ /*
+ * If it was previously non-full, then it's in some (possibly now
+ * incorrect) bin already; remove it.
+ *
+ * TODO: We bailed out early above if we didn't expand the longest free
+ * range, which should avoid a lot of redundant remove/reinserts in the
+ * same bin. But it doesn't eliminate all of them; it's possible that
+ * we decreased the longest free range length, but only slightly, and
+ * not enough to change our pszind. We could check that more precisely.
+ * (Or, ideally, size class dequantization will happen at some point,
+ * and the issue becomes moot).
+ */
+ if (ps_old_longest_free_range > 0) {
+ pszind_t old_pind = sz_psz2ind(sz_psz_quantize_floor(
+ ps_old_longest_free_range<< LG_PAGE));
+ edata_heap_remove(&psset->pageslabs[old_pind], ps);
+ if (edata_heap_empty(&psset->pageslabs[old_pind])) {
+ bitmap_set(psset->bitmap, &psset_bitmap_info,
+ (size_t)old_pind);
+ }
+ }
+ /* If the pageslab is empty, it gets evicted from the set. */
+ if (new_range_len == ps_npages) {
+ return ps;
+ }
+ /* Otherwise, it gets reinserted. */
+ pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor(
+ new_range_len << LG_PAGE));
+ if (edata_heap_empty(&psset->pageslabs[new_pind])) {
+ bitmap_unset(psset->bitmap, &psset_bitmap_info,
+ (size_t)new_pind);
+ }
+ edata_heap_insert(&psset->pageslabs[new_pind], ps);
+ return NULL;
+}
diff --git a/test/unit/psset.c b/test/unit/psset.c
new file mode 100644
index 00000000..8a5090d3
--- /dev/null
+++ b/test/unit/psset.c
@@ -0,0 +1,306 @@
+#include "test/jemalloc_test.h"
+
+#include "jemalloc/internal/psset.h"
+
+#define PAGESLAB_PAGES 64
+#define PAGESLAB_SIZE (PAGESLAB_PAGES << LG_PAGE)
+#define PAGESLAB_SN 123
+#define PAGESLAB_ADDR ((void *)(1234 << LG_PAGE))
+
+#define ALLOC_ARENA_IND 111
+#define ALLOC_ESN 222
+
+static void
+edata_init_test(edata_t *edata) {
+ memset(edata, 0, sizeof(*edata));
+ edata_arena_ind_set(edata, ALLOC_ARENA_IND);
+ edata_esn_set(edata, ALLOC_ESN);
+}
+
+static void
+edata_expect(edata_t *edata, size_t page_offset, size_t page_cnt) {
+ /*
+ * Note that allocations should get the arena ind of their home
+ * arena, *not* the arena ind of the pageslab allocator.
+ */
+ expect_u_eq(ALLOC_ARENA_IND, edata_arena_ind_get(edata),
+ "Arena ind changed");
+ expect_ptr_eq(
+ (void *)((uintptr_t)PAGESLAB_ADDR + (page_offset << LG_PAGE)),
+ edata_addr_get(edata), "Didn't allocate in order");
+ expect_zu_eq(page_cnt << LG_PAGE, edata_size_get(edata), "");
+ expect_false(edata_slab_get(edata), "");
+ expect_u_eq(SC_NSIZES, edata_szind_get_maybe_invalid(edata),
+ "");
+ expect_zu_eq(0, edata_sn_get(edata), "");
+ expect_d_eq(edata_state_get(edata), extent_state_active, "");
+ expect_false(edata_zeroed_get(edata), "");
+ expect_true(edata_committed_get(edata), "");
+ expect_d_eq(EXTENT_PAI_HPA, edata_pai_get(edata), "");
+ expect_false(edata_is_head_get(edata), "");
+}
+
+TEST_BEGIN(test_empty) {
+ bool err;
+ edata_t pageslab;
+ memset(&pageslab, 0, sizeof(pageslab));
+ edata_t alloc;
+
+ edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
+ /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
+ /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
+ EXTENT_IS_HEAD);
+ edata_init_test(&alloc);
+
+ psset_t psset;
+ psset_init(&psset);
+
+ /* Empty psset should return fail allocations. */
+ err = psset_alloc_reuse(&psset, &alloc, PAGE);
+ expect_true(err, "Empty psset succeeded in an allocation.");
+}
+TEST_END
+
+TEST_BEGIN(test_fill) {
+ bool err;
+ edata_t pageslab;
+ memset(&pageslab, 0, sizeof(pageslab));
+ edata_t alloc[PAGESLAB_PAGES];
+
+ edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
+ /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
+ /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
+ EXTENT_IS_HEAD);
+
+ psset_t psset;
+ psset_init(&psset);
+
+ edata_init_test(&alloc[0]);
+ psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
+ for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+ edata_init_test(&alloc[i]);
+ err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
+ expect_false(err, "Nonempty psset failed page allocation.");
+ }
+
+ for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ edata_t *edata = &alloc[i];
+ edata_expect(edata, i, 1);
+ }
+
+ /* The pageslab, and thus psset, should now have no allocations. */
+ edata_t extra_alloc;
+ edata_init_test(&extra_alloc);
+ err = psset_alloc_reuse(&psset, &extra_alloc, PAGE);
+ expect_true(err, "Alloc succeeded even though psset should be empty");
+}
+TEST_END
+
+TEST_BEGIN(test_reuse) {
+ bool err;
+ edata_t *ps;
+
+ edata_t pageslab;
+ memset(&pageslab, 0, sizeof(pageslab));
+ edata_t alloc[PAGESLAB_PAGES];
+
+ edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
+ /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
+ /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
+ EXTENT_IS_HEAD);
+
+ psset_t psset;
+ psset_init(&psset);
+
+ edata_init_test(&alloc[0]);
+ psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
+ for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+ edata_init_test(&alloc[i]);
+ err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
+ expect_false(err, "Nonempty psset failed page allocation.");
+ }
+
+ /* Free odd indices. */
+ for (size_t i = 0; i < PAGESLAB_PAGES; i ++) {
+ if (i % 2 == 0) {
+ continue;
+ }
+ ps = psset_dalloc(&psset, &alloc[i]);
+ expect_ptr_null(ps, "Nonempty pageslab evicted");
+ }
+ /* Realloc into them. */
+ for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ if (i % 2 == 0) {
+ continue;
+ }
+ err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
+ expect_false(err, "Nonempty psset failed page allocation.");
+ edata_expect(&alloc[i], i, 1);
+ }
+ /* Now, free the pages at indices 0 or 1 mod 2. */
+ for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ if (i % 4 > 1) {
+ continue;
+ }
+ ps = psset_dalloc(&psset, &alloc[i]);
+ expect_ptr_null(ps, "Nonempty pageslab evicted");
+ }
+ /* And realloc 2-page allocations into them. */
+ for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ if (i % 4 != 0) {
+ continue;
+ }
+ err = psset_alloc_reuse(&psset, &alloc[i], 2 * PAGE);
+ expect_false(err, "Nonempty psset failed page allocation.");
+ edata_expect(&alloc[i], i, 2);
+ }
+ /* Free all the 2-page allocations. */
+ for (size_t i = 0; i < PAGESLAB_PAGES; i++) {
+ if (i % 4 != 0) {
+ continue;
+ }
+ ps = psset_dalloc(&psset, &alloc[i]);
+ expect_ptr_null(ps, "Nonempty pageslab evicted");
+ }
+ /*
+ * Free up a 1-page hole next to a 2-page hole, but somewhere in the
+ * middle of the pageslab. Index 11 should be right before such a hole
+ * (since 12 % 4 == 0).
+ */
+ size_t index_of_3 = 11;
+ ps = psset_dalloc(&psset, &alloc[index_of_3]);
+ expect_ptr_null(ps, "Nonempty pageslab evicted");
+ err = psset_alloc_reuse(&psset, &alloc[index_of_3], 3 * PAGE);
+ expect_false(err, "Should have been able to find alloc.");
+ edata_expect(&alloc[index_of_3], index_of_3, 3);
+
+ /* Free up a 4-page hole at the end. */
+ ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]);
+ expect_ptr_null(ps, "Nonempty pageslab evicted");
+ ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 2]);
+ expect_ptr_null(ps, "Nonempty pageslab evicted");
+
+ /* Make sure we can satisfy an allocation at the very end of a slab. */
+ size_t index_of_4 = PAGESLAB_PAGES - 4;
+ ps = psset_dalloc(&psset, &alloc[index_of_4]);
+ expect_ptr_null(ps, "Nonempty pageslab evicted");
+ err = psset_alloc_reuse(&psset, &alloc[index_of_4], 4 * PAGE);
+ expect_false(err, "Should have been able to find alloc.");
+ edata_expect(&alloc[index_of_4], index_of_4, 4);
+}
+TEST_END
+
+TEST_BEGIN(test_evict) {
+ bool err;
+ edata_t *ps;
+ edata_t pageslab;
+ memset(&pageslab, 0, sizeof(pageslab));
+ edata_t alloc[PAGESLAB_PAGES];
+
+ edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
+ /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
+ /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
+ EXTENT_IS_HEAD);
+ psset_t psset;
+ psset_init(&psset);
+
+ /* Alloc the whole slab. */
+ edata_init_test(&alloc[0]);
+ psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
+ for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+ edata_init_test(&alloc[i]);
+ err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
+ expect_false(err, "Unxpected allocation failure");
+ }
+
+ /* Dealloc the whole slab, going forwards. */
+ for (size_t i = 0; i < PAGESLAB_PAGES - 1; i++) {
+ ps = psset_dalloc(&psset, &alloc[i]);
+ expect_ptr_null(ps, "Nonempty pageslab evicted");
+ }
+ ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]);
+ expect_ptr_eq(&pageslab, ps, "Empty pageslab not evicted.");
+
+ err = psset_alloc_reuse(&psset, &alloc[0], PAGE);
+ expect_true(err, "psset should be empty.");
+}
+TEST_END
+
+TEST_BEGIN(test_multi_pageslab) {
+ bool err;
+ edata_t *ps;
+ edata_t pageslab[2];
+ memset(&pageslab, 0, sizeof(pageslab));
+ edata_t alloc[2][PAGESLAB_PAGES];
+
+ edata_init(&pageslab[0], /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
+ /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
+ /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
+ EXTENT_IS_HEAD);
+ edata_init(&pageslab[1], /* arena_ind */ 0,
+ (void *)((uintptr_t)PAGESLAB_ADDR + PAGESLAB_SIZE), PAGESLAB_SIZE,
+ /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
+ /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
+ EXTENT_IS_HEAD);
+
+ psset_t psset;
+ psset_init(&psset);
+
+ /* Insert both slabs. */
+ edata_init_test(&alloc[0][0]);
+ psset_alloc_new(&psset, &pageslab[0], &alloc[0][0], PAGE);
+ edata_init_test(&alloc[1][0]);
+ psset_alloc_new(&psset, &pageslab[1], &alloc[1][0], PAGE);
+
+ /* Fill them both up; make sure we do so in first-fit order. */
+ for (size_t i = 0; i < 2; i++) {
+ for (size_t j = 1; j < PAGESLAB_PAGES; j++) {
+ edata_init_test(&alloc[i][j]);
+ err = psset_alloc_reuse(&psset, &alloc[i][j], PAGE);
+ expect_false(err,
+ "Nonempty psset failed page allocation.");
+ assert_ptr_eq(&pageslab[i], edata_ps_get(&alloc[i][j]),
+ "Didn't pick pageslabs in first-fit");
+ }
+ }
+
+ /*
+ * Free up a 2-page hole in the earlier slab, and a 1-page one in the
+ * later one. We should still pick the earlier slab for a 1-page
+ * allocation.
+ */
+ ps = psset_dalloc(&psset, &alloc[0][0]);
+ expect_ptr_null(ps, "Unexpected eviction");
+ ps = psset_dalloc(&psset, &alloc[0][1]);
+ expect_ptr_null(ps, "Unexpected eviction");
+ ps = psset_dalloc(&psset, &alloc[1][0]);
+ expect_ptr_null(ps, "Unexpected eviction");
+ err = psset_alloc_reuse(&psset, &alloc[0][0], PAGE);
+ expect_ptr_eq(&pageslab[0], edata_ps_get(&alloc[0][0]),
+ "Should have picked first pageslab");
+
+ /*
+ * Now both slabs have 1-page holes. Free up a second one in the later
+ * slab.
+ */
+ ps = psset_dalloc(&psset, &alloc[1][1]);
+ expect_ptr_null(ps, "Unexpected eviction");
+
+ /*
+ * We should be able to allocate a 2-page object, even though an earlier
+ * size class is nonempty.
+ */
+ err = psset_alloc_reuse(&psset, &alloc[1][0], 2 * PAGE);
+ expect_false(err, "Allocation should have succeeded");
+}
+TEST_END
+
+int
+main(void) {
+ return test_no_reentrancy(
+ test_empty,
+ test_fill,
+ test_reuse,
+ test_evict,
+ test_multi_pageslab);
+}