psset: Purge empty slabs first.

These are particularly good candidates for purging (listed in the diff).
This commit is contained in:
David Goldblatt 2021-06-03 16:21:29 -07:00 committed by David Goldblatt
parent 41fd56605e
commit 47d8a7e6b0
3 changed files with 143 additions and 7 deletions

View File

@ -25,6 +25,9 @@
* index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind + * index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind +
* 1). This lets us implement a preference for purging non-hugified hpdatas * 1). This lets us implement a preference for purging non-hugified hpdatas
* among similarly-dirty ones. * among similarly-dirty ones.
* We reserve the last two indices for empty slabs, in that case purging
* hugified ones (which are definitionally all waste) before non-hugified ones
* (i.e. reversing the order).
*/ */
#define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES) #define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES)
@ -78,7 +81,11 @@ struct psset_s {
* allocations. * allocations.
*/ */
hpdata_empty_list_t empty; hpdata_empty_list_t empty;
/* Slabs which are available to be purged, ordered by purge level. */ /*
* Slabs which are available to be purged, ordered by how much we want
* to purge them (with later indices indicating slabs we want to purge
* more).
*/
hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS]; hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS];
/* Bitmap for which set bits correspond to non-empty purge lists. */ /* Bitmap for which set bits correspond to non-empty purge lists. */
fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)]; fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)];

View File

@ -201,11 +201,32 @@ psset_purge_list_ind(hpdata_t *ps) {
size_t ndirty = hpdata_ndirty_get(ps); size_t ndirty = hpdata_ndirty_get(ps);
/* Shouldn't have something with no dirty pages purgeable. */ /* Shouldn't have something with no dirty pages purgeable. */
assert(ndirty > 0); assert(ndirty > 0);
/*
* Higher indices correspond to lists we'd like to purge earlier; make
* the two highest indices correspond to empty lists, which we attempt
* to purge before purging any non-empty list. This has two advantages:
* - Empty page slabs are the least likely to get reused (we'll only
* pick them for an allocation if we have no other choice).
* - Empty page slabs can purge every dirty page they contain in a
* single call, which is not usually the case.
*
* We purge hugeified empty slabs before nonhugeified ones, on the basis
* that they are fully dirty, while nonhugified slabs might not be, so
* we free up more pages more easily.
*/
if (hpdata_nactive_get(ps) == 0) {
if (hpdata_huge_get(ps)) {
return PSSET_NPURGE_LISTS - 1;
} else {
return PSSET_NPURGE_LISTS - 2;
}
}
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(ndirty << LG_PAGE)); pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(ndirty << LG_PAGE));
/* /*
* Higher indices correspond to lists we'd like to purge earlier; * For non-empty slabs, we may reuse them again. Prefer purging
* increment the index for the nonhugified hpdatas first, so that we'll * non-hugeified slabs before hugeified ones then, among pages of
* pick them before picking hugified ones. * similar dirtiness. We still get some benefit from the hugification.
*/ */
return (size_t)pind * 2 + (hpdata_huge_get(ps) ? 0 : 1); return (size_t)pind * 2 + (hpdata_huge_get(ps) ? 0 : 1);
} }
@ -321,7 +342,7 @@ psset_pick_purge(psset_t *psset) {
return NULL; return NULL;
} }
pszind_t ind = (pszind_t)ind_ssz; pszind_t ind = (pszind_t)ind_ssz;
assert(ind < PSSET_NPSIZES); assert(ind < PSSET_NPURGE_LISTS);
hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]); hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]);
assert(ps != NULL); assert(ps != NULL);
return ps; return ps;

View File

@ -545,7 +545,7 @@ TEST_END
TEST_BEGIN(test_purge_prefers_nonhuge) { TEST_BEGIN(test_purge_prefers_nonhuge) {
/* /*
* All else being equal, we should prefer purging non-huge pages over * All else being equal, we should prefer purging non-huge pages over
* huge ones. * huge ones for non-empty extents.
*/ */
/* Nothing magic about this constant. */ /* Nothing magic about this constant. */
@ -625,6 +625,112 @@ TEST_BEGIN(test_purge_prefers_nonhuge) {
} }
TEST_END TEST_END
TEST_BEGIN(test_purge_prefers_empty) {
void *ptr;
psset_t psset;
psset_init(&psset);
hpdata_t hpdata_empty;
hpdata_t hpdata_nonempty;
hpdata_init(&hpdata_empty, (void *)(10 * HUGEPAGE), 123);
psset_insert(&psset, &hpdata_empty);
hpdata_init(&hpdata_nonempty, (void *)(11 * HUGEPAGE), 456);
psset_insert(&psset, &hpdata_nonempty);
psset_update_begin(&psset, &hpdata_empty);
ptr = hpdata_reserve_alloc(&hpdata_empty, PAGE);
expect_ptr_eq(hpdata_addr_get(&hpdata_empty), ptr, "");
hpdata_unreserve(&hpdata_empty, ptr, PAGE);
hpdata_purge_allowed_set(&hpdata_empty, true);
psset_update_end(&psset, &hpdata_empty);
psset_update_begin(&psset, &hpdata_nonempty);
ptr = hpdata_reserve_alloc(&hpdata_nonempty, 10 * PAGE);
expect_ptr_eq(hpdata_addr_get(&hpdata_nonempty), ptr, "");
hpdata_unreserve(&hpdata_nonempty, ptr, 9 * PAGE);
hpdata_purge_allowed_set(&hpdata_nonempty, true);
psset_update_end(&psset, &hpdata_nonempty);
/*
* The nonempty slab has 9 dirty pages, while the empty one has only 1.
* We should still pick the empty one for purging.
*/
hpdata_t *to_purge = psset_pick_purge(&psset);
expect_ptr_eq(&hpdata_empty, to_purge, "");
}
TEST_END
TEST_BEGIN(test_purge_prefers_empty_huge) {
void *ptr;
psset_t psset;
psset_init(&psset);
enum {NHP = 10 };
hpdata_t hpdata_huge[NHP];
hpdata_t hpdata_nonhuge[NHP];
uintptr_t cur_addr = 100 * HUGEPAGE;
uint64_t cur_age = 123;
for (int i = 0; i < NHP; i++) {
hpdata_init(&hpdata_huge[i], (void *)cur_addr, cur_age);
cur_addr += HUGEPAGE;
cur_age++;
psset_insert(&psset, &hpdata_huge[i]);
hpdata_init(&hpdata_nonhuge[i], (void *)cur_addr, cur_age);
cur_addr += HUGEPAGE;
cur_age++;
psset_insert(&psset, &hpdata_nonhuge[i]);
/*
* Make the hpdata_huge[i] fully dirty, empty, purgable, and
* huge.
*/
psset_update_begin(&psset, &hpdata_huge[i]);
ptr = hpdata_reserve_alloc(&hpdata_huge[i], HUGEPAGE);
expect_ptr_eq(hpdata_addr_get(&hpdata_huge[i]), ptr, "");
hpdata_hugify(&hpdata_huge[i]);
hpdata_unreserve(&hpdata_huge[i], ptr, HUGEPAGE);
hpdata_purge_allowed_set(&hpdata_huge[i], true);
psset_update_end(&psset, &hpdata_huge[i]);
/*
* Make hpdata_nonhuge[i] fully dirty, empty, purgable, and
* non-huge.
*/
psset_update_begin(&psset, &hpdata_nonhuge[i]);
ptr = hpdata_reserve_alloc(&hpdata_nonhuge[i], HUGEPAGE);
expect_ptr_eq(hpdata_addr_get(&hpdata_nonhuge[i]), ptr, "");
hpdata_unreserve(&hpdata_nonhuge[i], ptr, HUGEPAGE);
hpdata_purge_allowed_set(&hpdata_nonhuge[i], true);
psset_update_end(&psset, &hpdata_nonhuge[i]);
}
/*
* We have a bunch of empty slabs, half huge, half nonhuge, inserted in
* alternating order. We should pop all the huge ones before popping
* any of the non-huge ones for purging.
*/
for (int i = 0; i < NHP; i++) {
hpdata_t *to_purge = psset_pick_purge(&psset);
expect_ptr_eq(&hpdata_huge[i], to_purge, "");
psset_update_begin(&psset, to_purge);
hpdata_purge_allowed_set(to_purge, false);
psset_update_end(&psset, to_purge);
}
for (int i = 0; i < NHP; i++) {
hpdata_t *to_purge = psset_pick_purge(&psset);
expect_ptr_eq(&hpdata_nonhuge[i], to_purge, "");
psset_update_begin(&psset, to_purge);
hpdata_purge_allowed_set(to_purge, false);
psset_update_end(&psset, to_purge);
}
}
TEST_END
int int
main(void) { main(void) {
return test_no_reentrancy( return test_no_reentrancy(
@ -636,5 +742,7 @@ main(void) {
test_stats, test_stats,
test_oldest_fit, test_oldest_fit,
test_insert_remove, test_insert_remove,
test_purge_prefers_nonhuge); test_purge_prefers_nonhuge,
test_purge_prefers_empty,
test_purge_prefers_empty_huge);
} }