project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

psset: Add insert/remove functions.

Browse Source 
These will allow us to (for instance) move pageslabs from a psset dedicated to
not-yet-hugeified pages to one dedicated to hugeified ones.
This commit is contained in:
David Goldblatt 2020-11-09 17:24:31 -08:00 committed by David Goldblatt
parent d438296b1f
commit d0a991d47b
3 changed files with 144 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/jemalloc/internal/psset.h
View File

@ -59,6 +59,9 @@ struct psset_s {
void psset_init(psset_t *psset);
void psset_insert(psset_t *psset, edata_t *ps);
void psset_remove(psset_t *psset, edata_t *ps);
/*
* Tries to obtain a chunk from an existing pageslab already in the set.
* Returns true on failure.

62
src/psset.c
View File

@ -65,6 +65,51 @@ psset_assert_ps_consistent(edata_t *ps) {
edata_size_get(ps) >> LG_PAGE) == edata_longest_free_range_get(ps));
}
void
psset_insert(psset_t *psset, edata_t *ps) {
psset_assert_ps_consistent(ps);
size_t longest_free_range = edata_longest_free_range_get(ps);
if (longest_free_range == 0) {
/*
* We don't ned to track full slabs; just pretend to for stats
* purposes. See the comment at psset_bin_stats_adjust.
*/
psset_bin_stats_adjust(&psset->full_slab_stats, ps,
/* inc */ true);
return;
}
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
longest_free_range << LG_PAGE));
assert(pind < PSSET_NPSIZES);
if (edata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
}
psset_edata_heap_insert(psset, pind, ps);
}
void
psset_remove(psset_t *psset, edata_t *ps) {
psset_assert_ps_consistent(ps);
size_t longest_free_range = edata_longest_free_range_get(ps);
if (longest_free_range == 0) {
psset_bin_stats_adjust(&psset->full_slab_stats, ps,
/* inc */ true);
return;
}
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
longest_free_range << LG_PAGE));
assert(pind < PSSET_NPSIZES);
psset_edata_heap_remove(psset, pind, ps);
if (edata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind);
}
}
/*
* Similar to PAC's extent_recycle_extract. Out of all the pageslabs in the
* set, picks one that can satisfy the allocation and remove it from the set.
@ -91,21 +136,6 @@ psset_recycle_extract(psset_t *psset, size_t size) {
return ps;
}
static void
psset_insert(psset_t *psset, edata_t *ps, size_t largest_range) {
psset_assert_ps_consistent(ps);
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
largest_range << LG_PAGE));
assert(pind < PSSET_NPSIZES);
if (edata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
}
psset_edata_heap_insert(psset, pind, ps);
}
/*
* Given a pageslab ps and an edata to allocate size bytes from, initializes the
* edata with a range in the pageslab, and puts ps back in the set.
@ -187,7 +217,7 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
psset_bin_stats_adjust(&psset->full_slab_stats, ps,
/* inc */ true);
} else {
psset_insert(psset, ps, largest_unchosen_range);
psset_insert(psset, ps);
}
}

121
test/unit/psset.c
View File

@ -360,23 +360,37 @@ TEST_BEGIN(test_stats) {
expect_false(err, "Nonempty psset failed page allocation.");
}
stats_expect(&psset, PAGESLAB_PAGES);
edata_t *ps;
for (ssize_t i = PAGESLAB_PAGES - 1; i >= 0; i--) {
edata_t *ps = psset_dalloc(&psset, &alloc[i]);
ps = psset_dalloc(&psset, &alloc[i]);
expect_true((ps == NULL) == (i != 0),
"psset_dalloc should only evict a slab on the last free");
stats_expect(&psset, i);
}
psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
stats_expect(&psset, 1);
psset_remove(&psset, &pageslab);
stats_expect(&psset, 0);
psset_insert(&psset, &pageslab);
stats_expect(&psset, 1);
}
TEST_END
TEST_BEGIN(test_oldest_fit) {
/*
* Fills in and inserts two pageslabs, with the first better than the second,
* and each fully allocated (into the allocations in allocs and worse_allocs,
* each of which should be PAGESLAB_PAGES long).
*
* (There's nothing magic about these numbers; it's just useful to share the
* setup between the oldest fit and the insert/remove test).
*/
static void
init_test_pageslabs(psset_t *psset, edata_t *pageslab, edata_t *worse_pageslab,
edata_t *alloc, edata_t *worse_alloc) {
bool err;
edata_t alloc[PAGESLAB_PAGES];
edata_t worse_alloc[PAGESLAB_PAGES];
edata_t pageslab;
memset(&pageslab, 0, sizeof(pageslab));
edata_init(&pageslab, /* arena_ind */ 0, (void *)(10 * PAGESLAB_SIZE),
memset(pageslab, 0, sizeof(*pageslab));
edata_init(pageslab, /* arena_ind */ 0, (void *)(10 * PAGESLAB_SIZE),
PAGESLAB_SIZE, /* slab */ true, SC_NSIZES, PAGESLAB_SN + 1,
extent_state_active, /* zeroed */ false, /* comitted */ true,
EXTENT_PAI_HPA, EXTENT_IS_HEAD);
@ -386,29 +400,27 @@ TEST_BEGIN(test_oldest_fit) {
* added to the set after the previous one, and so should be less
* preferred for allocations.
*/
edata_t worse_pageslab;
memset(&worse_pageslab, 0, sizeof(pageslab));
edata_init(&worse_pageslab, /* arena_ind */ 0,
memset(worse_pageslab, 0, sizeof(*worse_pageslab));
edata_init(worse_pageslab, /* arena_ind */ 0,
(void *)(9 * PAGESLAB_SIZE), PAGESLAB_SIZE, /* slab */ true,
SC_NSIZES, PAGESLAB_SN - 1, extent_state_active, /* zeroed */ false,
/* comitted */ true, EXTENT_PAI_HPA, EXTENT_IS_HEAD);
psset_t psset;
psset_init(&psset);
psset_init(psset);
edata_init_test(&alloc[0]);
psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
psset_alloc_new(psset, pageslab, &alloc[0], PAGE);
for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
edata_init_test(&alloc[i]);
err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
err = psset_alloc_reuse(psset, &alloc[i], PAGE);
expect_false(err, "Nonempty psset failed page allocation.");
expect_ptr_eq(&pageslab, edata_ps_get(&alloc[i]),
expect_ptr_eq(pageslab, edata_ps_get(&alloc[i]),
"Allocated from the wrong pageslab");
}
edata_init_test(&worse_alloc[0]);
psset_alloc_new(&psset, &worse_pageslab, &worse_alloc[0], PAGE);
expect_ptr_eq(&worse_pageslab, edata_ps_get(&worse_alloc[0]),
psset_alloc_new(psset, worse_pageslab, &worse_alloc[0], PAGE);
expect_ptr_eq(worse_pageslab, edata_ps_get(&worse_alloc[0]),
"Allocated from the wrong pageslab");
/*
* Make the two pssets otherwise indistinguishable; all full except for
@ -416,20 +428,31 @@ TEST_BEGIN(test_oldest_fit) {
*/
for (size_t i = 1; i < PAGESLAB_PAGES - 1; i++) {
edata_init_test(&worse_alloc[i]);
err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
err = psset_alloc_reuse(psset, &alloc[i], PAGE);
expect_false(err, "Nonempty psset failed page allocation.");
expect_ptr_eq(&worse_pageslab, edata_ps_get(&alloc[i]),
expect_ptr_eq(worse_pageslab, edata_ps_get(&alloc[i]),
"Allocated from the wrong pageslab");
}
/* Deallocate the last page from the older pageslab. */
edata_t *evicted = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]);
edata_t *evicted = psset_dalloc(psset, &alloc[PAGESLAB_PAGES - 1]);
expect_ptr_null(evicted, "Unexpected eviction");
}
/*
* This edata is the whole purpose for the test; it should come from the
* older pageslab.
*/
TEST_BEGIN(test_oldest_fit) {
bool err;
edata_t alloc[PAGESLAB_PAGES];
edata_t worse_alloc[PAGESLAB_PAGES];
edata_t pageslab;
edata_t worse_pageslab;
psset_t psset;
init_test_pageslabs(&psset, &pageslab, &worse_pageslab, alloc,
worse_alloc);
/* The edata should come from the better pageslab. */
edata_t test_edata;
edata_init_test(&test_edata);
err = psset_alloc_reuse(&psset, &test_edata, PAGE);
@ -439,6 +462,51 @@ TEST_BEGIN(test_oldest_fit) {
}
TEST_END
TEST_BEGIN(test_insert_remove) {
bool err;
edata_t *ps;
edata_t alloc[PAGESLAB_PAGES];
edata_t worse_alloc[PAGESLAB_PAGES];
edata_t pageslab;
edata_t worse_pageslab;
psset_t psset;
init_test_pageslabs(&psset, &pageslab, &worse_pageslab, alloc,
worse_alloc);
/* Remove better; should still be able to alloc from worse. */
psset_remove(&psset, &pageslab);
err = psset_alloc_reuse(&psset, &worse_alloc[PAGESLAB_PAGES - 1], PAGE);
expect_false(err, "Removal should still leave an empty page");
expect_ptr_eq(&worse_pageslab,
edata_ps_get(&worse_alloc[PAGESLAB_PAGES - 1]),
"Allocated out of wrong ps");
/*
* After deallocating the previous alloc and reinserting better, it
* should be preferred for future allocations.
*/
ps = psset_dalloc(&psset, &worse_alloc[PAGESLAB_PAGES - 1]);
expect_ptr_null(ps, "Incorrect eviction of nonempty pageslab");
psset_insert(&psset, &pageslab);
err = psset_alloc_reuse(&psset, &alloc[PAGESLAB_PAGES - 1], PAGE);
expect_false(err, "psset should be nonempty");
expect_ptr_eq(&pageslab, edata_ps_get(&alloc[PAGESLAB_PAGES - 1]),
"Removal/reinsertion shouldn't change ordering");
/*
* After deallocating and removing both, allocations should fail.
*/
ps = psset_dalloc(&psset, &alloc[PAGESLAB_PAGES - 1]);
expect_ptr_null(ps, "Incorrect eviction");
psset_remove(&psset, &pageslab);
psset_remove(&psset, &worse_pageslab);
err = psset_alloc_reuse(&psset, &alloc[PAGESLAB_PAGES - 1], PAGE);
expect_true(err, "psset should be empty, but an alloc succeeded");
}
TEST_END
int
main(void) {
return test_no_reentrancy(
@ -448,5 +516,6 @@ main(void) {
test_evict,
test_multi_pageslab,
test_stats,
test_oldest_fit);
test_oldest_fit,
test_insert_remove);
}