From 47d8a7e6b04a81f2938f1b18f66cb468870fa442 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Thu, 3 Jun 2021 16:21:29 -0700 Subject: [PATCH] psset: Purge empty slabs first. These are particularly good candidates for purging (listed in the diff). --- include/jemalloc/internal/psset.h | 9 ++- src/psset.c | 29 ++++++-- test/unit/psset.c | 112 +++++++++++++++++++++++++++++- 3 files changed, 143 insertions(+), 7 deletions(-) diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index 96fb300e..e1d64970 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -25,6 +25,9 @@ * index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind + * 1). This lets us implement a preference for purging non-hugified hpdatas * among similarly-dirty ones. + * We reserve the last two indices for empty slabs, in that case purging + * hugified ones (which are definitionally all waste) before non-hugified ones + * (i.e. reversing the order). */ #define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES) @@ -78,7 +81,11 @@ struct psset_s { * allocations. */ hpdata_empty_list_t empty; - /* Slabs which are available to be purged, ordered by purge level. */ + /* + * Slabs which are available to be purged, ordered by how much we want + * to purge them (with later indices indicating slabs we want to purge + * more). + */ hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS]; /* Bitmap for which set bits correspond to non-empty purge lists. */ fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)]; diff --git a/src/psset.c b/src/psset.c index 5978202a..9a8f054f 100644 --- a/src/psset.c +++ b/src/psset.c @@ -201,11 +201,32 @@ psset_purge_list_ind(hpdata_t *ps) { size_t ndirty = hpdata_ndirty_get(ps); /* Shouldn't have something with no dirty pages purgeable. */ assert(ndirty > 0); + /* + * Higher indices correspond to lists we'd like to purge earlier; make + * the two highest indices correspond to empty lists, which we attempt + * to purge before purging any non-empty list. This has two advantages: + * - Empty page slabs are the least likely to get reused (we'll only + * pick them for an allocation if we have no other choice). + * - Empty page slabs can purge every dirty page they contain in a + * single call, which is not usually the case. + * + * We purge hugeified empty slabs before nonhugeified ones, on the basis + * that they are fully dirty, while nonhugified slabs might not be, so + * we free up more pages more easily. + */ + if (hpdata_nactive_get(ps) == 0) { + if (hpdata_huge_get(ps)) { + return PSSET_NPURGE_LISTS - 1; + } else { + return PSSET_NPURGE_LISTS - 2; + } + } + pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(ndirty << LG_PAGE)); /* - * Higher indices correspond to lists we'd like to purge earlier; - * increment the index for the nonhugified hpdatas first, so that we'll - * pick them before picking hugified ones. + * For non-empty slabs, we may reuse them again. Prefer purging + * non-hugeified slabs before hugeified ones then, among pages of + * similar dirtiness. We still get some benefit from the hugification. */ return (size_t)pind * 2 + (hpdata_huge_get(ps) ? 0 : 1); } @@ -321,7 +342,7 @@ psset_pick_purge(psset_t *psset) { return NULL; } pszind_t ind = (pszind_t)ind_ssz; - assert(ind < PSSET_NPSIZES); + assert(ind < PSSET_NPURGE_LISTS); hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]); assert(ps != NULL); return ps; diff --git a/test/unit/psset.c b/test/unit/psset.c index 7bce7c1b..6ff72012 100644 --- a/test/unit/psset.c +++ b/test/unit/psset.c @@ -545,7 +545,7 @@ TEST_END TEST_BEGIN(test_purge_prefers_nonhuge) { /* * All else being equal, we should prefer purging non-huge pages over - * huge ones. + * huge ones for non-empty extents. */ /* Nothing magic about this constant. */ @@ -625,6 +625,112 @@ TEST_BEGIN(test_purge_prefers_nonhuge) { } TEST_END +TEST_BEGIN(test_purge_prefers_empty) { + void *ptr; + + psset_t psset; + psset_init(&psset); + + hpdata_t hpdata_empty; + hpdata_t hpdata_nonempty; + hpdata_init(&hpdata_empty, (void *)(10 * HUGEPAGE), 123); + psset_insert(&psset, &hpdata_empty); + hpdata_init(&hpdata_nonempty, (void *)(11 * HUGEPAGE), 456); + psset_insert(&psset, &hpdata_nonempty); + + psset_update_begin(&psset, &hpdata_empty); + ptr = hpdata_reserve_alloc(&hpdata_empty, PAGE); + expect_ptr_eq(hpdata_addr_get(&hpdata_empty), ptr, ""); + hpdata_unreserve(&hpdata_empty, ptr, PAGE); + hpdata_purge_allowed_set(&hpdata_empty, true); + psset_update_end(&psset, &hpdata_empty); + + psset_update_begin(&psset, &hpdata_nonempty); + ptr = hpdata_reserve_alloc(&hpdata_nonempty, 10 * PAGE); + expect_ptr_eq(hpdata_addr_get(&hpdata_nonempty), ptr, ""); + hpdata_unreserve(&hpdata_nonempty, ptr, 9 * PAGE); + hpdata_purge_allowed_set(&hpdata_nonempty, true); + psset_update_end(&psset, &hpdata_nonempty); + + /* + * The nonempty slab has 9 dirty pages, while the empty one has only 1. + * We should still pick the empty one for purging. + */ + hpdata_t *to_purge = psset_pick_purge(&psset); + expect_ptr_eq(&hpdata_empty, to_purge, ""); +} +TEST_END + +TEST_BEGIN(test_purge_prefers_empty_huge) { + void *ptr; + + psset_t psset; + psset_init(&psset); + + enum {NHP = 10 }; + + hpdata_t hpdata_huge[NHP]; + hpdata_t hpdata_nonhuge[NHP]; + + uintptr_t cur_addr = 100 * HUGEPAGE; + uint64_t cur_age = 123; + for (int i = 0; i < NHP; i++) { + hpdata_init(&hpdata_huge[i], (void *)cur_addr, cur_age); + cur_addr += HUGEPAGE; + cur_age++; + psset_insert(&psset, &hpdata_huge[i]); + + hpdata_init(&hpdata_nonhuge[i], (void *)cur_addr, cur_age); + cur_addr += HUGEPAGE; + cur_age++; + psset_insert(&psset, &hpdata_nonhuge[i]); + + /* + * Make the hpdata_huge[i] fully dirty, empty, purgable, and + * huge. + */ + psset_update_begin(&psset, &hpdata_huge[i]); + ptr = hpdata_reserve_alloc(&hpdata_huge[i], HUGEPAGE); + expect_ptr_eq(hpdata_addr_get(&hpdata_huge[i]), ptr, ""); + hpdata_hugify(&hpdata_huge[i]); + hpdata_unreserve(&hpdata_huge[i], ptr, HUGEPAGE); + hpdata_purge_allowed_set(&hpdata_huge[i], true); + psset_update_end(&psset, &hpdata_huge[i]); + + /* + * Make hpdata_nonhuge[i] fully dirty, empty, purgable, and + * non-huge. + */ + psset_update_begin(&psset, &hpdata_nonhuge[i]); + ptr = hpdata_reserve_alloc(&hpdata_nonhuge[i], HUGEPAGE); + expect_ptr_eq(hpdata_addr_get(&hpdata_nonhuge[i]), ptr, ""); + hpdata_unreserve(&hpdata_nonhuge[i], ptr, HUGEPAGE); + hpdata_purge_allowed_set(&hpdata_nonhuge[i], true); + psset_update_end(&psset, &hpdata_nonhuge[i]); + } + + /* + * We have a bunch of empty slabs, half huge, half nonhuge, inserted in + * alternating order. We should pop all the huge ones before popping + * any of the non-huge ones for purging. + */ + for (int i = 0; i < NHP; i++) { + hpdata_t *to_purge = psset_pick_purge(&psset); + expect_ptr_eq(&hpdata_huge[i], to_purge, ""); + psset_update_begin(&psset, to_purge); + hpdata_purge_allowed_set(to_purge, false); + psset_update_end(&psset, to_purge); + } + for (int i = 0; i < NHP; i++) { + hpdata_t *to_purge = psset_pick_purge(&psset); + expect_ptr_eq(&hpdata_nonhuge[i], to_purge, ""); + psset_update_begin(&psset, to_purge); + hpdata_purge_allowed_set(to_purge, false); + psset_update_end(&psset, to_purge); + } +} +TEST_END + int main(void) { return test_no_reentrancy( @@ -636,5 +742,7 @@ main(void) { test_stats, test_oldest_fit, test_insert_remove, - test_purge_prefers_nonhuge); + test_purge_prefers_nonhuge, + test_purge_prefers_empty, + test_purge_prefers_empty_huge); }