psset: Purge empty slabs first.
These are particularly good candidates for purging (listed in the diff).
This commit is contained in:
parent
41fd56605e
commit
47d8a7e6b0
@ -25,6 +25,9 @@
|
|||||||
* index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind +
|
* index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind +
|
||||||
* 1). This lets us implement a preference for purging non-hugified hpdatas
|
* 1). This lets us implement a preference for purging non-hugified hpdatas
|
||||||
* among similarly-dirty ones.
|
* among similarly-dirty ones.
|
||||||
|
* We reserve the last two indices for empty slabs, in that case purging
|
||||||
|
* hugified ones (which are definitionally all waste) before non-hugified ones
|
||||||
|
* (i.e. reversing the order).
|
||||||
*/
|
*/
|
||||||
#define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES)
|
#define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES)
|
||||||
|
|
||||||
@ -78,7 +81,11 @@ struct psset_s {
|
|||||||
* allocations.
|
* allocations.
|
||||||
*/
|
*/
|
||||||
hpdata_empty_list_t empty;
|
hpdata_empty_list_t empty;
|
||||||
/* Slabs which are available to be purged, ordered by purge level. */
|
/*
|
||||||
|
* Slabs which are available to be purged, ordered by how much we want
|
||||||
|
* to purge them (with later indices indicating slabs we want to purge
|
||||||
|
* more).
|
||||||
|
*/
|
||||||
hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS];
|
hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS];
|
||||||
/* Bitmap for which set bits correspond to non-empty purge lists. */
|
/* Bitmap for which set bits correspond to non-empty purge lists. */
|
||||||
fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)];
|
fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)];
|
||||||
|
29
src/psset.c
29
src/psset.c
@ -201,11 +201,32 @@ psset_purge_list_ind(hpdata_t *ps) {
|
|||||||
size_t ndirty = hpdata_ndirty_get(ps);
|
size_t ndirty = hpdata_ndirty_get(ps);
|
||||||
/* Shouldn't have something with no dirty pages purgeable. */
|
/* Shouldn't have something with no dirty pages purgeable. */
|
||||||
assert(ndirty > 0);
|
assert(ndirty > 0);
|
||||||
|
/*
|
||||||
|
* Higher indices correspond to lists we'd like to purge earlier; make
|
||||||
|
* the two highest indices correspond to empty lists, which we attempt
|
||||||
|
* to purge before purging any non-empty list. This has two advantages:
|
||||||
|
* - Empty page slabs are the least likely to get reused (we'll only
|
||||||
|
* pick them for an allocation if we have no other choice).
|
||||||
|
* - Empty page slabs can purge every dirty page they contain in a
|
||||||
|
* single call, which is not usually the case.
|
||||||
|
*
|
||||||
|
* We purge hugeified empty slabs before nonhugeified ones, on the basis
|
||||||
|
* that they are fully dirty, while nonhugified slabs might not be, so
|
||||||
|
* we free up more pages more easily.
|
||||||
|
*/
|
||||||
|
if (hpdata_nactive_get(ps) == 0) {
|
||||||
|
if (hpdata_huge_get(ps)) {
|
||||||
|
return PSSET_NPURGE_LISTS - 1;
|
||||||
|
} else {
|
||||||
|
return PSSET_NPURGE_LISTS - 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(ndirty << LG_PAGE));
|
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(ndirty << LG_PAGE));
|
||||||
/*
|
/*
|
||||||
* Higher indices correspond to lists we'd like to purge earlier;
|
* For non-empty slabs, we may reuse them again. Prefer purging
|
||||||
* increment the index for the nonhugified hpdatas first, so that we'll
|
* non-hugeified slabs before hugeified ones then, among pages of
|
||||||
* pick them before picking hugified ones.
|
* similar dirtiness. We still get some benefit from the hugification.
|
||||||
*/
|
*/
|
||||||
return (size_t)pind * 2 + (hpdata_huge_get(ps) ? 0 : 1);
|
return (size_t)pind * 2 + (hpdata_huge_get(ps) ? 0 : 1);
|
||||||
}
|
}
|
||||||
@ -321,7 +342,7 @@ psset_pick_purge(psset_t *psset) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
pszind_t ind = (pszind_t)ind_ssz;
|
pszind_t ind = (pszind_t)ind_ssz;
|
||||||
assert(ind < PSSET_NPSIZES);
|
assert(ind < PSSET_NPURGE_LISTS);
|
||||||
hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]);
|
hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]);
|
||||||
assert(ps != NULL);
|
assert(ps != NULL);
|
||||||
return ps;
|
return ps;
|
||||||
|
@ -545,7 +545,7 @@ TEST_END
|
|||||||
TEST_BEGIN(test_purge_prefers_nonhuge) {
|
TEST_BEGIN(test_purge_prefers_nonhuge) {
|
||||||
/*
|
/*
|
||||||
* All else being equal, we should prefer purging non-huge pages over
|
* All else being equal, we should prefer purging non-huge pages over
|
||||||
* huge ones.
|
* huge ones for non-empty extents.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Nothing magic about this constant. */
|
/* Nothing magic about this constant. */
|
||||||
@ -625,6 +625,112 @@ TEST_BEGIN(test_purge_prefers_nonhuge) {
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
|
TEST_BEGIN(test_purge_prefers_empty) {
|
||||||
|
void *ptr;
|
||||||
|
|
||||||
|
psset_t psset;
|
||||||
|
psset_init(&psset);
|
||||||
|
|
||||||
|
hpdata_t hpdata_empty;
|
||||||
|
hpdata_t hpdata_nonempty;
|
||||||
|
hpdata_init(&hpdata_empty, (void *)(10 * HUGEPAGE), 123);
|
||||||
|
psset_insert(&psset, &hpdata_empty);
|
||||||
|
hpdata_init(&hpdata_nonempty, (void *)(11 * HUGEPAGE), 456);
|
||||||
|
psset_insert(&psset, &hpdata_nonempty);
|
||||||
|
|
||||||
|
psset_update_begin(&psset, &hpdata_empty);
|
||||||
|
ptr = hpdata_reserve_alloc(&hpdata_empty, PAGE);
|
||||||
|
expect_ptr_eq(hpdata_addr_get(&hpdata_empty), ptr, "");
|
||||||
|
hpdata_unreserve(&hpdata_empty, ptr, PAGE);
|
||||||
|
hpdata_purge_allowed_set(&hpdata_empty, true);
|
||||||
|
psset_update_end(&psset, &hpdata_empty);
|
||||||
|
|
||||||
|
psset_update_begin(&psset, &hpdata_nonempty);
|
||||||
|
ptr = hpdata_reserve_alloc(&hpdata_nonempty, 10 * PAGE);
|
||||||
|
expect_ptr_eq(hpdata_addr_get(&hpdata_nonempty), ptr, "");
|
||||||
|
hpdata_unreserve(&hpdata_nonempty, ptr, 9 * PAGE);
|
||||||
|
hpdata_purge_allowed_set(&hpdata_nonempty, true);
|
||||||
|
psset_update_end(&psset, &hpdata_nonempty);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The nonempty slab has 9 dirty pages, while the empty one has only 1.
|
||||||
|
* We should still pick the empty one for purging.
|
||||||
|
*/
|
||||||
|
hpdata_t *to_purge = psset_pick_purge(&psset);
|
||||||
|
expect_ptr_eq(&hpdata_empty, to_purge, "");
|
||||||
|
}
|
||||||
|
TEST_END
|
||||||
|
|
||||||
|
TEST_BEGIN(test_purge_prefers_empty_huge) {
|
||||||
|
void *ptr;
|
||||||
|
|
||||||
|
psset_t psset;
|
||||||
|
psset_init(&psset);
|
||||||
|
|
||||||
|
enum {NHP = 10 };
|
||||||
|
|
||||||
|
hpdata_t hpdata_huge[NHP];
|
||||||
|
hpdata_t hpdata_nonhuge[NHP];
|
||||||
|
|
||||||
|
uintptr_t cur_addr = 100 * HUGEPAGE;
|
||||||
|
uint64_t cur_age = 123;
|
||||||
|
for (int i = 0; i < NHP; i++) {
|
||||||
|
hpdata_init(&hpdata_huge[i], (void *)cur_addr, cur_age);
|
||||||
|
cur_addr += HUGEPAGE;
|
||||||
|
cur_age++;
|
||||||
|
psset_insert(&psset, &hpdata_huge[i]);
|
||||||
|
|
||||||
|
hpdata_init(&hpdata_nonhuge[i], (void *)cur_addr, cur_age);
|
||||||
|
cur_addr += HUGEPAGE;
|
||||||
|
cur_age++;
|
||||||
|
psset_insert(&psset, &hpdata_nonhuge[i]);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Make the hpdata_huge[i] fully dirty, empty, purgable, and
|
||||||
|
* huge.
|
||||||
|
*/
|
||||||
|
psset_update_begin(&psset, &hpdata_huge[i]);
|
||||||
|
ptr = hpdata_reserve_alloc(&hpdata_huge[i], HUGEPAGE);
|
||||||
|
expect_ptr_eq(hpdata_addr_get(&hpdata_huge[i]), ptr, "");
|
||||||
|
hpdata_hugify(&hpdata_huge[i]);
|
||||||
|
hpdata_unreserve(&hpdata_huge[i], ptr, HUGEPAGE);
|
||||||
|
hpdata_purge_allowed_set(&hpdata_huge[i], true);
|
||||||
|
psset_update_end(&psset, &hpdata_huge[i]);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Make hpdata_nonhuge[i] fully dirty, empty, purgable, and
|
||||||
|
* non-huge.
|
||||||
|
*/
|
||||||
|
psset_update_begin(&psset, &hpdata_nonhuge[i]);
|
||||||
|
ptr = hpdata_reserve_alloc(&hpdata_nonhuge[i], HUGEPAGE);
|
||||||
|
expect_ptr_eq(hpdata_addr_get(&hpdata_nonhuge[i]), ptr, "");
|
||||||
|
hpdata_unreserve(&hpdata_nonhuge[i], ptr, HUGEPAGE);
|
||||||
|
hpdata_purge_allowed_set(&hpdata_nonhuge[i], true);
|
||||||
|
psset_update_end(&psset, &hpdata_nonhuge[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We have a bunch of empty slabs, half huge, half nonhuge, inserted in
|
||||||
|
* alternating order. We should pop all the huge ones before popping
|
||||||
|
* any of the non-huge ones for purging.
|
||||||
|
*/
|
||||||
|
for (int i = 0; i < NHP; i++) {
|
||||||
|
hpdata_t *to_purge = psset_pick_purge(&psset);
|
||||||
|
expect_ptr_eq(&hpdata_huge[i], to_purge, "");
|
||||||
|
psset_update_begin(&psset, to_purge);
|
||||||
|
hpdata_purge_allowed_set(to_purge, false);
|
||||||
|
psset_update_end(&psset, to_purge);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < NHP; i++) {
|
||||||
|
hpdata_t *to_purge = psset_pick_purge(&psset);
|
||||||
|
expect_ptr_eq(&hpdata_nonhuge[i], to_purge, "");
|
||||||
|
psset_update_begin(&psset, to_purge);
|
||||||
|
hpdata_purge_allowed_set(to_purge, false);
|
||||||
|
psset_update_end(&psset, to_purge);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TEST_END
|
||||||
|
|
||||||
int
|
int
|
||||||
main(void) {
|
main(void) {
|
||||||
return test_no_reentrancy(
|
return test_no_reentrancy(
|
||||||
@ -636,5 +742,7 @@ main(void) {
|
|||||||
test_stats,
|
test_stats,
|
||||||
test_oldest_fit,
|
test_oldest_fit,
|
||||||
test_insert_remove,
|
test_insert_remove,
|
||||||
test_purge_prefers_nonhuge);
|
test_purge_prefers_nonhuge,
|
||||||
|
test_purge_prefers_empty,
|
||||||
|
test_purge_prefers_empty_huge);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user