Add hpa_central module
This will be the centralized component of the coming hugepage allocator; the source of larger chunks of memory from which smaller ones can be obtained.
This commit is contained in:
David Goldblatt
committed by
David Goldblatt
David Goldblatt
parent
1ed7ec369f
commit
21b70cb540
450
test/unit/hpa_central.c
Normal file
450
test/unit/hpa_central.c
Normal file
@@ -0,0 +1,450 @@
|
||||
#include "test/jemalloc_test.h"
|
||||
|
||||
#include "jemalloc/internal/hpa_central.h"
|
||||
|
||||
typedef struct test_data_s test_data_t;
|
||||
struct test_data_s {
|
||||
/*
|
||||
* Must be the first member -- we convert back and forth between the
|
||||
* test_data_t and the hpa_central_t;
|
||||
*/
|
||||
hpa_central_t central;
|
||||
base_t *base;
|
||||
edata_cache_t edata_cache;
|
||||
emap_t emap;
|
||||
};
|
||||
|
||||
void
|
||||
create_test_data(hpa_central_t **r_central, base_t **r_base) {
|
||||
bool err;
|
||||
base_t *base = base_new(TSDN_NULL, /* ind */ 111,
|
||||
&ehooks_default_extent_hooks);
|
||||
assert_ptr_not_null(base, "");
|
||||
|
||||
test_data_t *test_data = malloc(sizeof(test_data_t));
|
||||
assert_ptr_not_null(test_data, "");
|
||||
|
||||
test_data->base = base;
|
||||
|
||||
err = edata_cache_init(&test_data->edata_cache, base);
|
||||
assert_false(err, "");
|
||||
|
||||
err = emap_init(&test_data->emap, test_data->base,
|
||||
/* zeroed */ false);
|
||||
assert_false(err, "");
|
||||
|
||||
hpa_central_init(&test_data->central, &test_data->edata_cache,
|
||||
&test_data->emap);
|
||||
|
||||
*r_central = (hpa_central_t *)test_data;
|
||||
*r_base = base;
|
||||
}
|
||||
|
||||
static void
|
||||
destroy_test_data(hpa_central_t *central) {
|
||||
test_data_t *test_data = (test_data_t *)central;
|
||||
base_delete(TSDN_NULL, test_data->base);
|
||||
free(test_data);
|
||||
}
|
||||
|
||||
static edata_t *
|
||||
test_edata(base_t *base, uintptr_t addr, size_t size) {
|
||||
edata_t *edata = base_alloc_edata(TSDN_NULL, base);
|
||||
assert_ptr_not_null(edata, "");
|
||||
edata_init(edata, base_ind_get(base), (void *)addr,
|
||||
size, /* slab */ false, /* szind_t */ SC_NSIZES, /* sn */ 0,
|
||||
extent_state_active, /* zeroed */ true, /* comitted */ true,
|
||||
EXTENT_PAI_HPA, /* is_head */ true);
|
||||
return edata;
|
||||
}
|
||||
|
||||
static void
|
||||
edata_expect_alloc(base_t *base, edata_t *edata, uintptr_t addr, size_t size) {
|
||||
expect_ptr_not_null(edata, "Alloc should have succeeded");
|
||||
expect_u_eq(base_ind_get(base), edata_arena_ind_get(edata), "");
|
||||
expect_u_eq(SC_NSIZES, edata_szind_get_maybe_invalid(edata), "");
|
||||
expect_d_eq(extent_state_active, edata_state_get(edata), "");
|
||||
assert_ptr_eq((void *)addr, edata_base_get(edata), "");
|
||||
assert_zu_eq(size, edata_size_get(edata), "");
|
||||
}
|
||||
|
||||
|
||||
TEST_BEGIN(test_empty) {
|
||||
hpa_central_t *central;
|
||||
base_t *base;
|
||||
create_test_data(¢ral, &base);
|
||||
|
||||
edata_t *edata;
|
||||
|
||||
edata = hpa_central_alloc_reuse(TSDN_NULL, central, PAGE, PAGE);
|
||||
expect_ptr_null(edata, "Empty allocator succeed in its allocation");
|
||||
|
||||
edata = hpa_central_alloc_reuse(TSDN_NULL, central, PAGE, 2 * PAGE);
|
||||
expect_ptr_null(edata, "Empty allocator succeed in its allocation");
|
||||
|
||||
edata = hpa_central_alloc_reuse(TSDN_NULL, central, PAGE, 8 * PAGE);
|
||||
expect_ptr_null(edata, "Empty allocator succeed in its allocation");
|
||||
|
||||
edata = hpa_central_alloc_reuse(TSDN_NULL, central, 4 * PAGE, 8 * PAGE);
|
||||
expect_ptr_null(edata, "Empty allocator succeed in its allocation");
|
||||
|
||||
destroy_test_data(central);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_first_fit_simple) {
|
||||
hpa_central_t *central;
|
||||
base_t *base;
|
||||
create_test_data(¢ral, &base);
|
||||
|
||||
edata_t *edata1 = test_edata(base, 10 * PAGE, 10 * PAGE);
|
||||
bool err = hpa_central_alloc_grow(TSDN_NULL, central, PAGE, edata1);
|
||||
expect_false(err, "Unexpected grow failure");
|
||||
edata_expect_alloc(base, edata1, 10 * PAGE, PAGE);
|
||||
|
||||
edata_t *edata2 = test_edata(base, 4 * PAGE, 1 * PAGE);
|
||||
err = hpa_central_alloc_grow(TSDN_NULL, central, PAGE, edata2);
|
||||
expect_false(err, "Unexpected grow failure");
|
||||
edata_expect_alloc(base, edata2, 4 * PAGE, PAGE);
|
||||
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata2);
|
||||
|
||||
/*
|
||||
* Even though there's a lower-addressed extent that a by-size search
|
||||
* will find earlier, we should still pick the earlier one.
|
||||
*/
|
||||
edata_t *edata3 = hpa_central_alloc_reuse(TSDN_NULL, central, PAGE, PAGE);
|
||||
/*
|
||||
* Recall there's still an active page at the beginning of the extent
|
||||
* added at 10 * PAGE; the next allocation from it should be at 11 *
|
||||
* PAGE.
|
||||
*/
|
||||
edata_expect_alloc(base, edata3, 11 * PAGE, PAGE);
|
||||
|
||||
destroy_test_data(central);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_first_fit_large_goal) {
|
||||
/*
|
||||
* See the comment in hpa_central_alloc_reuse; we should prefer an
|
||||
* earlier allocation over a later one, even if it means we fall short
|
||||
* of the goal size.
|
||||
*/
|
||||
hpa_central_t *central;
|
||||
base_t *base;
|
||||
create_test_data(¢ral, &base);
|
||||
|
||||
edata_t *edata1 = test_edata(base, 10 * PAGE, 10 * PAGE);
|
||||
bool err = hpa_central_alloc_grow(TSDN_NULL, central, 2 * PAGE, edata1);
|
||||
expect_false(err, "Unexpected grow failure");
|
||||
edata_expect_alloc(base, edata1, 10 * PAGE, 2 * PAGE);
|
||||
|
||||
/* We need a page, but would like 2. */
|
||||
edata_t *edata2 = hpa_central_alloc_reuse(TSDN_NULL, central, PAGE,
|
||||
2 * PAGE);
|
||||
edata_expect_alloc(base, edata2, 12 * PAGE, 2 * PAGE);
|
||||
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata1);
|
||||
|
||||
/*
|
||||
* Now, we have a 2-page inactive extent, then a 2-page active extent,
|
||||
* then a 6-page inactive extent. If our minimum size is 2 but the goal
|
||||
* size is 4, we should still pick the first hole rather than the
|
||||
* second.
|
||||
*/
|
||||
edata1 = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE, 4 * PAGE);
|
||||
edata_expect_alloc(base, edata1, 10 * PAGE, 2 * PAGE);
|
||||
|
||||
/*
|
||||
* Make sure we didn't succeed only by forgetting about that last range
|
||||
* or something.
|
||||
*/
|
||||
edata_t *edata3 = hpa_central_alloc_reuse(TSDN_NULL, central, 4 * PAGE,
|
||||
4 * PAGE);
|
||||
edata_expect_alloc(base, edata3, 14 * PAGE, 4 * PAGE);
|
||||
|
||||
destroy_test_data(central);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_merging) {
|
||||
hpa_central_t *central;
|
||||
base_t *base;
|
||||
create_test_data(¢ral, &base);
|
||||
|
||||
/* Test an exact match */
|
||||
bool err;
|
||||
edata_t *edata1 = test_edata(base, 10 * PAGE, PAGE);
|
||||
err = hpa_central_alloc_grow(TSDN_NULL, central, PAGE, edata1);
|
||||
expect_false(err, "Alloc should have succeeded");
|
||||
edata_expect_alloc(base, edata1, 10 * PAGE, PAGE);
|
||||
|
||||
edata_t *edata2 = hpa_central_alloc_reuse(TSDN_NULL, central, PAGE,
|
||||
PAGE);
|
||||
expect_ptr_null(edata2, "Allocation should have failed");
|
||||
|
||||
/*
|
||||
* Create two more regions; one immediately before the first and one
|
||||
* immediately after. The extents shouldn't get merged.
|
||||
*/
|
||||
edata2 = test_edata(base, 11 * PAGE, PAGE);
|
||||
err = hpa_central_alloc_grow(TSDN_NULL, central, PAGE, edata2);
|
||||
edata_expect_alloc(base, edata2, 11 * PAGE, PAGE);
|
||||
|
||||
edata_t *edata3 = test_edata(base, 12 * PAGE, 20 * PAGE);
|
||||
err = hpa_central_alloc_grow(TSDN_NULL, central, PAGE, edata3);
|
||||
edata_expect_alloc(base, edata3, 12 * PAGE, PAGE);
|
||||
|
||||
/*
|
||||
* OK, we've got 3 contiguous ranges; [10, 11), [11, 12), and [12, 22).
|
||||
* They shouldn't get merged though, even once freed. We free the
|
||||
* middle range last to test merging (or rather, the lack thereof) in
|
||||
* both directions.
|
||||
*/
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata1);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata3);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata2);
|
||||
|
||||
/*
|
||||
* A two-page range should only be satisfied by the third added region.
|
||||
*/
|
||||
edata_t *edata = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE,
|
||||
2 * PAGE);
|
||||
edata_expect_alloc(base, edata, 12 * PAGE, 2 * PAGE);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata);
|
||||
|
||||
/* Same with a three-page range. */
|
||||
edata = hpa_central_alloc_reuse(TSDN_NULL, central, 3 * PAGE, 3 * PAGE);
|
||||
edata_expect_alloc(base, edata, 12 * PAGE, 3 * PAGE);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata);
|
||||
|
||||
/* Let's try some cases that *should* get merged. */
|
||||
edata1 = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE, 2 * PAGE);
|
||||
edata_expect_alloc(base, edata1, 12 * PAGE, 2 * PAGE);
|
||||
edata2 = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE, 2 * PAGE);
|
||||
edata_expect_alloc(base, edata2, 14 * PAGE, 2 * PAGE);
|
||||
edata3 = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE, 2 * PAGE);
|
||||
edata_expect_alloc(base, edata3, 16 * PAGE, 2 * PAGE);
|
||||
|
||||
/* Merge with predecessor. */
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata1);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata2);
|
||||
edata1 = hpa_central_alloc_reuse(TSDN_NULL, central, 4 * PAGE,
|
||||
4 * PAGE);
|
||||
edata_expect_alloc(base, edata1, 12 * PAGE, 4 * PAGE);
|
||||
|
||||
/* Merge with successor */
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata3);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata1);
|
||||
edata1 = hpa_central_alloc_reuse(TSDN_NULL, central, 6 * PAGE,
|
||||
6 * PAGE);
|
||||
edata_expect_alloc(base, edata1, 12 * PAGE, 6 * PAGE);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata1);
|
||||
|
||||
/*
|
||||
* Let's try merging with both. We need to get three adjacent
|
||||
* allocations again; do it the same way as before.
|
||||
*/
|
||||
edata1 = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE, 2 * PAGE);
|
||||
edata_expect_alloc(base, edata1, 12 * PAGE, 2 * PAGE);
|
||||
edata2 = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE, 2 * PAGE);
|
||||
edata_expect_alloc(base, edata2, 14 * PAGE, 2 * PAGE);
|
||||
edata3 = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE, 2 * PAGE);
|
||||
edata_expect_alloc(base, edata3, 16 * PAGE, 2 * PAGE);
|
||||
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata1);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata3);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edata2);
|
||||
|
||||
edata1 = hpa_central_alloc_reuse(TSDN_NULL, central, 6 * PAGE,
|
||||
6 * PAGE);
|
||||
edata_expect_alloc(base, edata1, 12 * PAGE, 6 * PAGE);
|
||||
|
||||
destroy_test_data(central);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_stress_simple) {
|
||||
hpa_central_t *central;
|
||||
base_t *base;
|
||||
create_test_data(¢ral, &base);
|
||||
|
||||
enum {
|
||||
range_base = 1024 * PAGE,
|
||||
range_pages = 256,
|
||||
range_size = range_pages * PAGE
|
||||
};
|
||||
|
||||
edata_t *edatas[range_pages];
|
||||
|
||||
bool err;
|
||||
edata_t *range = test_edata(base, range_base, range_size);
|
||||
err = hpa_central_alloc_grow(TSDN_NULL, central, PAGE, range);
|
||||
expect_false(err, "Unexpected grow failure");
|
||||
hpa_central_dalloc(TSDN_NULL, central, range);
|
||||
|
||||
for (size_t i = 0; i < range_pages; i++) {
|
||||
edatas[i] = hpa_central_alloc_reuse(TSDN_NULL, central, PAGE,
|
||||
PAGE);
|
||||
edata_expect_alloc(base, edatas[i], range_base + i * PAGE,
|
||||
PAGE);
|
||||
}
|
||||
/* Free up the odd indices. */
|
||||
for (size_t i = 0; i < range_pages; i++) {
|
||||
if (i % 2 == 0) {
|
||||
continue;
|
||||
}
|
||||
hpa_central_dalloc(TSDN_NULL, central, edatas[i]);
|
||||
}
|
||||
/*
|
||||
* Reallocate them again. Try it with a goal size that can't be
|
||||
* satisfied.
|
||||
*/
|
||||
for (size_t i = 0; i < range_pages; i++) {
|
||||
if (i % 2 == 0) {
|
||||
continue;
|
||||
}
|
||||
edatas[i] = hpa_central_alloc_reuse(TSDN_NULL, central, PAGE,
|
||||
PAGE);
|
||||
edata_expect_alloc(base, edatas[i], range_base + i * PAGE,
|
||||
PAGE);
|
||||
}
|
||||
/*
|
||||
* In each batch of 8, create a free range of 4 pages and a free range
|
||||
* of 2 pages.
|
||||
*/
|
||||
for (size_t i = 0; i < range_pages; i += 8) {
|
||||
hpa_central_dalloc(TSDN_NULL, central, edatas[i + 1]);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edatas[i + 2]);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edatas[i + 3]);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edatas[i + 4]);
|
||||
|
||||
hpa_central_dalloc(TSDN_NULL, central, edatas[i + 6]);
|
||||
hpa_central_dalloc(TSDN_NULL, central, edatas[i + 7]);
|
||||
}
|
||||
|
||||
/*
|
||||
* And allocate 3 pages into the first, and 2 pages into the second. To
|
||||
* mix things up a little, lets get those amounts via goal sizes
|
||||
* instead.
|
||||
*/
|
||||
for (size_t i = 0; i < range_pages; i += 8) {
|
||||
edatas[i + 1] = hpa_central_alloc_reuse(TSDN_NULL, central,
|
||||
2 * PAGE, 3 * PAGE);
|
||||
edata_expect_alloc(base, edatas[i + 1],
|
||||
range_base + (i + 1) * PAGE, 3 * PAGE);
|
||||
|
||||
edatas[i + 6] = hpa_central_alloc_reuse(TSDN_NULL, central,
|
||||
2 * PAGE, 4 * PAGE);
|
||||
edata_expect_alloc(base, edatas[i + 6],
|
||||
range_base + (i + 6) * PAGE, 2 * PAGE);
|
||||
}
|
||||
|
||||
edata_t *edata = hpa_central_alloc_reuse(TSDN_NULL, central, 2 * PAGE,
|
||||
2 * PAGE);
|
||||
expect_ptr_null(edata, "Should be no free ranges of 2 pages");
|
||||
|
||||
destroy_test_data(central);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_stress_random) {
|
||||
const size_t range_length = 32 * PAGE;
|
||||
const size_t range_base = 100 * PAGE;
|
||||
const size_t size_max_pages = 16;
|
||||
|
||||
hpa_central_t *central;
|
||||
base_t *base;
|
||||
create_test_data(¢ral, &base);
|
||||
|
||||
/*
|
||||
* We loop through this once per some operations, so we don't want it to
|
||||
* get too big.
|
||||
*/
|
||||
const size_t nlive_edatas_max = 100;
|
||||
size_t nlive_edatas = 0;
|
||||
edata_t **live_edatas = calloc(nlive_edatas_max, sizeof(edata_t *));
|
||||
size_t nranges = 0;
|
||||
|
||||
/*
|
||||
* Nothing special about this constant; we're only fixing it for
|
||||
* consistency across runs.
|
||||
*/
|
||||
size_t prng_state = (size_t)0x76999ffb014df07c;
|
||||
for (size_t i = 0; i < 100 * 1000; i++) {
|
||||
size_t operation = prng_range_zu(&prng_state, 2);
|
||||
if (operation == 0) {
|
||||
/* Do an alloc. */
|
||||
if (nlive_edatas == nlive_edatas_max) {
|
||||
continue;
|
||||
}
|
||||
size_t min_pages = 1 + prng_range_zu(
|
||||
&prng_state, size_max_pages);
|
||||
size_t goal_pages = min_pages + prng_range_zu(
|
||||
&prng_state, size_max_pages - min_pages + 1);
|
||||
edata_t *edata = hpa_central_alloc_reuse(TSDN_NULL,
|
||||
central, min_pages * PAGE, goal_pages * PAGE);
|
||||
if (edata == NULL) {
|
||||
edata = test_edata(base,
|
||||
range_base + range_length * nranges,
|
||||
range_length);
|
||||
bool err = hpa_central_alloc_grow(TSDN_NULL,
|
||||
central, goal_pages * PAGE, edata);
|
||||
assert_false(err, "Unexpected grow failure");
|
||||
nranges++;
|
||||
}
|
||||
uintptr_t begin = (uintptr_t)edata_base_get(edata);
|
||||
uintptr_t end = (uintptr_t)edata_last_get(edata);
|
||||
size_t range_begin = (begin - range_base) / range_length;
|
||||
size_t range_end = (end - range_base) / range_length;
|
||||
expect_zu_eq(range_begin, range_end,
|
||||
"Should not have allocations spanning "
|
||||
"multiple ranges");
|
||||
expect_zu_ge(begin, range_base,
|
||||
"Gave back a pointer outside of the reserved "
|
||||
"range");
|
||||
expect_zu_lt(end, range_base + range_length * nranges,
|
||||
"Gave back a pointer outside of the reserved "
|
||||
"range");
|
||||
for (size_t j = 0; j < nlive_edatas; j++) {
|
||||
edata_t *other = live_edatas[j];
|
||||
uintptr_t other_begin =
|
||||
(uintptr_t)edata_base_get(other);
|
||||
uintptr_t other_end =
|
||||
(uintptr_t)edata_last_get(other);
|
||||
expect_true(
|
||||
(begin < other_begin && end < other_begin)
|
||||
|| (begin > other_end),
|
||||
"Gave back two extents that overlap");
|
||||
}
|
||||
live_edatas[nlive_edatas] = edata;
|
||||
nlive_edatas++;
|
||||
} else {
|
||||
/* Do a free. */
|
||||
if (nlive_edatas == 0) {
|
||||
continue;
|
||||
}
|
||||
size_t victim = prng_range_zu(&prng_state,
|
||||
nlive_edatas);
|
||||
edata_t *to_free = live_edatas[victim];
|
||||
live_edatas[victim] = live_edatas[nlive_edatas - 1];
|
||||
nlive_edatas--;
|
||||
hpa_central_dalloc(TSDN_NULL, central, to_free);
|
||||
}
|
||||
}
|
||||
|
||||
free(live_edatas);
|
||||
destroy_test_data(central);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
int main(void) {
|
||||
return test_no_reentrancy(
|
||||
test_empty,
|
||||
test_first_fit_simple,
|
||||
test_first_fit_large_goal,
|
||||
test_merging,
|
||||
test_stress_simple,
|
||||
test_stress_random);
|
||||
}
|
||||
Reference in New Issue
Block a user