From d0a991d47b2717ac6abe6a7d8adc52c967ecd115 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Mon, 9 Nov 2020 17:24:31 -0800 Subject: [PATCH] psset: Add insert/remove functions. These will allow us to (for instance) move pageslabs from a psset dedicated to not-yet-hugeified pages to one dedicated to hugeified ones. --- include/jemalloc/internal/psset.h | 3 + src/psset.c | 62 +++++++++++---- test/unit/psset.c | 121 +++++++++++++++++++++++------- 3 files changed, 144 insertions(+), 42 deletions(-) diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index 14311239..4b0c4da4 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -59,6 +59,9 @@ struct psset_s { void psset_init(psset_t *psset); +void psset_insert(psset_t *psset, edata_t *ps); +void psset_remove(psset_t *psset, edata_t *ps); + /* * Tries to obtain a chunk from an existing pageslab already in the set. * Returns true on failure. diff --git a/src/psset.c b/src/psset.c index 9fc7ec14..cd0dcae7 100644 --- a/src/psset.c +++ b/src/psset.c @@ -65,6 +65,51 @@ psset_assert_ps_consistent(edata_t *ps) { edata_size_get(ps) >> LG_PAGE) == edata_longest_free_range_get(ps)); } +void +psset_insert(psset_t *psset, edata_t *ps) { + psset_assert_ps_consistent(ps); + size_t longest_free_range = edata_longest_free_range_get(ps); + + if (longest_free_range == 0) { + /* + * We don't ned to track full slabs; just pretend to for stats + * purposes. See the comment at psset_bin_stats_adjust. + */ + psset_bin_stats_adjust(&psset->full_slab_stats, ps, + /* inc */ true); + return; + } + + pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( + longest_free_range << LG_PAGE)); + + assert(pind < PSSET_NPSIZES); + if (edata_age_heap_empty(&psset->pageslabs[pind])) { + bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind); + } + psset_edata_heap_insert(psset, pind, ps); +} + +void +psset_remove(psset_t *psset, edata_t *ps) { + psset_assert_ps_consistent(ps); + size_t longest_free_range = edata_longest_free_range_get(ps); + + if (longest_free_range == 0) { + psset_bin_stats_adjust(&psset->full_slab_stats, ps, + /* inc */ true); + return; + } + + pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( + longest_free_range << LG_PAGE)); + assert(pind < PSSET_NPSIZES); + psset_edata_heap_remove(psset, pind, ps); + if (edata_age_heap_empty(&psset->pageslabs[pind])) { + bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind); + } +} + /* * Similar to PAC's extent_recycle_extract. Out of all the pageslabs in the * set, picks one that can satisfy the allocation and remove it from the set. @@ -91,21 +136,6 @@ psset_recycle_extract(psset_t *psset, size_t size) { return ps; } -static void -psset_insert(psset_t *psset, edata_t *ps, size_t largest_range) { - psset_assert_ps_consistent(ps); - - pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( - largest_range << LG_PAGE)); - - assert(pind < PSSET_NPSIZES); - - if (edata_age_heap_empty(&psset->pageslabs[pind])) { - bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind); - } - psset_edata_heap_insert(psset, pind, ps); -} - /* * Given a pageslab ps and an edata to allocate size bytes from, initializes the * edata with a range in the pageslab, and puts ps back in the set. @@ -187,7 +217,7 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, psset_bin_stats_adjust(&psset->full_slab_stats, ps, /* inc */ true); } else { - psset_insert(psset, ps, largest_unchosen_range); + psset_insert(psset, ps); } } diff --git a/test/unit/psset.c b/test/unit/psset.c index 861903d6..e734ec8e 100644 --- a/test/unit/psset.c +++ b/test/unit/psset.c @@ -360,23 +360,37 @@ TEST_BEGIN(test_stats) { expect_false(err, "Nonempty psset failed page allocation."); } stats_expect(&psset, PAGESLAB_PAGES); + edata_t *ps; for (ssize_t i = PAGESLAB_PAGES - 1; i >= 0; i--) { - edata_t *ps = psset_dalloc(&psset, &alloc[i]); + ps = psset_dalloc(&psset, &alloc[i]); expect_true((ps == NULL) == (i != 0), "psset_dalloc should only evict a slab on the last free"); stats_expect(&psset, i); } + + psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE); + stats_expect(&psset, 1); + psset_remove(&psset, &pageslab); + stats_expect(&psset, 0); + psset_insert(&psset, &pageslab); + stats_expect(&psset, 1); } TEST_END -TEST_BEGIN(test_oldest_fit) { +/* + * Fills in and inserts two pageslabs, with the first better than the second, + * and each fully allocated (into the allocations in allocs and worse_allocs, + * each of which should be PAGESLAB_PAGES long). + * + * (There's nothing magic about these numbers; it's just useful to share the + * setup between the oldest fit and the insert/remove test). + */ +static void +init_test_pageslabs(psset_t *psset, edata_t *pageslab, edata_t *worse_pageslab, + edata_t *alloc, edata_t *worse_alloc) { bool err; - edata_t alloc[PAGESLAB_PAGES]; - edata_t worse_alloc[PAGESLAB_PAGES]; - - edata_t pageslab; - memset(&pageslab, 0, sizeof(pageslab)); - edata_init(&pageslab, /* arena_ind */ 0, (void *)(10 * PAGESLAB_SIZE), + memset(pageslab, 0, sizeof(*pageslab)); + edata_init(pageslab, /* arena_ind */ 0, (void *)(10 * PAGESLAB_SIZE), PAGESLAB_SIZE, /* slab */ true, SC_NSIZES, PAGESLAB_SN + 1, extent_state_active, /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, EXTENT_IS_HEAD); @@ -386,29 +400,27 @@ TEST_BEGIN(test_oldest_fit) { * added to the set after the previous one, and so should be less * preferred for allocations. */ - edata_t worse_pageslab; - memset(&worse_pageslab, 0, sizeof(pageslab)); - edata_init(&worse_pageslab, /* arena_ind */ 0, + memset(worse_pageslab, 0, sizeof(*worse_pageslab)); + edata_init(worse_pageslab, /* arena_ind */ 0, (void *)(9 * PAGESLAB_SIZE), PAGESLAB_SIZE, /* slab */ true, SC_NSIZES, PAGESLAB_SN - 1, extent_state_active, /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, EXTENT_IS_HEAD); - psset_t psset; - psset_init(&psset); + psset_init(psset); edata_init_test(&alloc[0]); - psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE); + psset_alloc_new(psset, pageslab, &alloc[0], PAGE); for (size_t i = 1; i < PAGESLAB_PAGES; i++) { edata_init_test(&alloc[i]); - err = psset_alloc_reuse(&psset, &alloc[i], PAGE); + err = psset_alloc_reuse(psset, &alloc[i], PAGE); expect_false(err, "Nonempty psset failed page allocation."); - expect_ptr_eq(&pageslab, edata_ps_get(&alloc[i]), + expect_ptr_eq(pageslab, edata_ps_get(&alloc[i]), "Allocated from the wrong pageslab"); } edata_init_test(&worse_alloc[0]); - psset_alloc_new(&psset, &worse_pageslab, &worse_alloc[0], PAGE); - expect_ptr_eq(&worse_pageslab, edata_ps_get(&worse_alloc[0]), + psset_alloc_new(psset, worse_pageslab, &worse_alloc[0], PAGE); + expect_ptr_eq(worse_pageslab, edata_ps_get(&worse_alloc[0]), "Allocated from the wrong pageslab"); /* * Make the two pssets otherwise indistinguishable; all full except for @@ -416,20 +428,31 @@ TEST_BEGIN(test_oldest_fit) { */ for (size_t i = 1; i < PAGESLAB_PAGES - 1; i++) { edata_init_test(&worse_alloc[i]); - err = psset_alloc_reuse(&psset, &alloc[i], PAGE); + err = psset_alloc_reuse(psset, &alloc[i], PAGE); expect_false(err, "Nonempty psset failed page allocation."); - expect_ptr_eq(&worse_pageslab, edata_ps_get(&alloc[i]), + expect_ptr_eq(worse_pageslab, edata_ps_get(&alloc[i]), "Allocated from the wrong pageslab"); } /* Deallocate the last page from the older pageslab. */ - edata_t *evicted = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]); + edata_t *evicted = psset_dalloc(psset, &alloc[PAGESLAB_PAGES - 1]); expect_ptr_null(evicted, "Unexpected eviction"); +} - /* - * This edata is the whole purpose for the test; it should come from the - * older pageslab. - */ +TEST_BEGIN(test_oldest_fit) { + bool err; + edata_t alloc[PAGESLAB_PAGES]; + edata_t worse_alloc[PAGESLAB_PAGES]; + + edata_t pageslab; + edata_t worse_pageslab; + + psset_t psset; + + init_test_pageslabs(&psset, &pageslab, &worse_pageslab, alloc, + worse_alloc); + + /* The edata should come from the better pageslab. */ edata_t test_edata; edata_init_test(&test_edata); err = psset_alloc_reuse(&psset, &test_edata, PAGE); @@ -439,6 +462,51 @@ TEST_BEGIN(test_oldest_fit) { } TEST_END +TEST_BEGIN(test_insert_remove) { + bool err; + edata_t *ps; + edata_t alloc[PAGESLAB_PAGES]; + edata_t worse_alloc[PAGESLAB_PAGES]; + + edata_t pageslab; + edata_t worse_pageslab; + + psset_t psset; + + init_test_pageslabs(&psset, &pageslab, &worse_pageslab, alloc, + worse_alloc); + + /* Remove better; should still be able to alloc from worse. */ + psset_remove(&psset, &pageslab); + err = psset_alloc_reuse(&psset, &worse_alloc[PAGESLAB_PAGES - 1], PAGE); + expect_false(err, "Removal should still leave an empty page"); + expect_ptr_eq(&worse_pageslab, + edata_ps_get(&worse_alloc[PAGESLAB_PAGES - 1]), + "Allocated out of wrong ps"); + + /* + * After deallocating the previous alloc and reinserting better, it + * should be preferred for future allocations. + */ + ps = psset_dalloc(&psset, &worse_alloc[PAGESLAB_PAGES - 1]); + expect_ptr_null(ps, "Incorrect eviction of nonempty pageslab"); + psset_insert(&psset, &pageslab); + err = psset_alloc_reuse(&psset, &alloc[PAGESLAB_PAGES - 1], PAGE); + expect_false(err, "psset should be nonempty"); + expect_ptr_eq(&pageslab, edata_ps_get(&alloc[PAGESLAB_PAGES - 1]), + "Removal/reinsertion shouldn't change ordering"); + /* + * After deallocating and removing both, allocations should fail. + */ + ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]); + expect_ptr_null(ps, "Incorrect eviction"); + psset_remove(&psset, &pageslab); + psset_remove(&psset, &worse_pageslab); + err = psset_alloc_reuse(&psset, &alloc[PAGESLAB_PAGES - 1], PAGE); + expect_true(err, "psset should be empty, but an alloc succeeded"); +} +TEST_END + int main(void) { return test_no_reentrancy( @@ -448,5 +516,6 @@ main(void) { test_evict, test_multi_pageslab, test_stats, - test_oldest_fit); + test_oldest_fit, + test_insert_remove); }