hpdata: Add state changing helpers.
We're about to allow hugepage subextent purging; get as much of our metadata handling ready as possible.
This commit is contained in:
parent
9b75808be1
commit
70692cfb13
@ -34,6 +34,16 @@ struct hpdata_s {
|
||||
uint64_t h_age;
|
||||
/* Whether or not we think the hugepage is mapped that way by the OS. */
|
||||
bool h_huge;
|
||||
|
||||
/*
|
||||
* Whether or not some thread is purging this hpdata (i.e. has called
|
||||
* hpdata_purge_begin but not yet called hpdata_purge_end), or
|
||||
* hugifying it. Only one thread at a time is allowed to change a
|
||||
* hugepage's state.
|
||||
*/
|
||||
bool h_mid_purge;
|
||||
bool h_mid_hugify;
|
||||
|
||||
union {
|
||||
/* When nonempty, used by the psset bins. */
|
||||
phn(hpdata_t) ph_link;
|
||||
@ -90,6 +100,22 @@ hpdata_huge_get(const hpdata_t *hpdata) {
|
||||
return hpdata->h_huge;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
hpdata_changing_state_get(const hpdata_t *hpdata) {
|
||||
return hpdata->h_mid_purge || hpdata->h_mid_hugify;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
hpdata_mid_purge_get(const hpdata_t *hpdata) {
|
||||
return hpdata->h_mid_purge;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
hpdata_mid_hugify_get(const hpdata_t *hpdata) {
|
||||
return hpdata->h_mid_hugify;
|
||||
}
|
||||
|
||||
|
||||
static inline size_t
|
||||
hpdata_longest_free_range_get(const hpdata_t *hpdata) {
|
||||
return hpdata->h_longest_free_range;
|
||||
@ -106,6 +132,11 @@ hpdata_nactive_get(hpdata_t *hpdata) {
|
||||
return hpdata->h_nactive;
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
hpdata_ndirty_get(hpdata_t *hpdata) {
|
||||
return hpdata->h_ndirty;
|
||||
}
|
||||
|
||||
static inline void
|
||||
hpdata_assert_empty(hpdata_t *hpdata) {
|
||||
assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES));
|
||||
@ -164,20 +195,69 @@ void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
|
||||
void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
|
||||
void hpdata_unreserve(hpdata_t *hpdata, void *begin, size_t sz);
|
||||
|
||||
/*
|
||||
* Tell the hpdata that it's now a hugepage (which, correspondingly, means that
|
||||
* all its pages become dirty.
|
||||
*/
|
||||
void hpdata_hugify(hpdata_t *hpdata);
|
||||
/*
|
||||
* Tell the hpdata that it's no longer a hugepage (all its pages are still
|
||||
* counted as dirty, though; an explicit purge call is required to change that).
|
||||
*/
|
||||
void hpdata_dehugify(hpdata_t *hpdata);
|
||||
/*
|
||||
* Tell the hpdata (which should be empty) that all dirty pages in it have been
|
||||
* purged.
|
||||
*/
|
||||
void hpdata_purge(hpdata_t *hpdata);
|
||||
|
||||
/*
|
||||
* The hpdata_purge_prepare_t allows grabbing the metadata required to purge
|
||||
* subranges of a hugepage while holding a lock, drop the lock during the actual
|
||||
* purging of them, and reacquire it to update the metadata again.
|
||||
*/
|
||||
typedef struct hpdata_purge_state_s hpdata_purge_state_t;
|
||||
struct hpdata_purge_state_s {
|
||||
size_t npurged;
|
||||
fb_group_t to_purge[FB_NGROUPS(HUGEPAGE_PAGES)];
|
||||
size_t next_purge_search_begin;
|
||||
};
|
||||
|
||||
/*
|
||||
* Initializes purge state. The access to hpdata must be externally
|
||||
* synchronized with other hpdata_* calls.
|
||||
*
|
||||
* You can tell whether or not a thread is purging or hugifying a given hpdata
|
||||
* via hpdata_changing_state_get(hpdata). Racing hugification or purging
|
||||
* operations aren't allowed.
|
||||
*
|
||||
* Once you begin purging, you have to follow through and call hpdata_purge_next
|
||||
* until you're done, and then end. Allocating out of an hpdata undergoing
|
||||
* purging is not allowed.
|
||||
*/
|
||||
void hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state);
|
||||
/*
|
||||
* If there are more extents to purge, sets *r_purge_addr and *r_purge_size to
|
||||
* true, and returns true. Otherwise, returns false to indicate that we're
|
||||
* done.
|
||||
*
|
||||
* This requires exclusive access to the purge state, but *not* to the hpdata.
|
||||
* In particular, unreserve calls are allowed while purging (i.e. you can dalloc
|
||||
* into one part of the hpdata while purging a different part).
|
||||
*/
|
||||
bool hpdata_purge_next(hpdata_t *hpdata, hpdata_purge_state_t *purge_state,
|
||||
void **r_purge_addr, size_t *r_purge_size);
|
||||
/*
|
||||
* Updates the hpdata metadata after all purging is done. Needs external
|
||||
* synchronization.
|
||||
*/
|
||||
void hpdata_purge_end(hpdata_t *hpdata, hpdata_purge_state_t *purge_state);
|
||||
|
||||
/*
|
||||
* Similarly, when hugifying , callers can do the metadata modifications while
|
||||
* holding a lock (thereby setting the change_state field), but actually do the
|
||||
* operation without blocking other threads.
|
||||
*/
|
||||
void hpdata_hugify_begin(hpdata_t *hpdata);
|
||||
void hpdata_hugify_end(hpdata_t *hpdata);
|
||||
|
||||
/*
|
||||
* Tell the hpdata that it's no longer a hugepage (all its pages are still
|
||||
* counted as dirty, though; an explicit purge call is required to change that).
|
||||
*
|
||||
* This should only be done after starting to purge, and before actually purging
|
||||
* any contents.
|
||||
*/
|
||||
void hpdata_dehugify(hpdata_t *hpdata);
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_HPDATA_H */
|
||||
|
@ -300,7 +300,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
|
||||
|
||||
bool hugify = hpa_should_hugify(shard, ps);
|
||||
if (hugify) {
|
||||
hpdata_hugify(ps);
|
||||
hpdata_hugify_begin(ps);
|
||||
}
|
||||
psset_insert(&shard->psset, ps);
|
||||
|
||||
@ -319,6 +319,9 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
|
||||
* operations in this hpa shard.
|
||||
*/
|
||||
hpa_hugify(ps);
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
hpdata_hugify_end(ps);
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
}
|
||||
return edata;
|
||||
}
|
||||
|
112
src/hpdata.c
112
src/hpdata.c
@ -22,6 +22,8 @@ hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
|
||||
hpdata_addr_set(hpdata, addr);
|
||||
hpdata_age_set(hpdata, age);
|
||||
hpdata->h_huge = false;
|
||||
hpdata->h_mid_purge = false;
|
||||
hpdata->h_mid_hugify = false;
|
||||
hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES);
|
||||
hpdata->h_nactive = 0;
|
||||
fb_init(hpdata->active_pages, HUGEPAGE_PAGES);
|
||||
@ -140,17 +142,125 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
|
||||
}
|
||||
|
||||
void
|
||||
hpdata_hugify(hpdata_t *hpdata) {
|
||||
hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
|
||||
hpdata_assert_consistent(hpdata);
|
||||
assert(!hpdata->h_mid_purge);
|
||||
assert(!hpdata->h_mid_hugify);
|
||||
hpdata->h_mid_purge = true;
|
||||
|
||||
purge_state->npurged = 0;
|
||||
purge_state->next_purge_search_begin = 0;
|
||||
|
||||
/*
|
||||
* Initialize to_purge with everything that's not active but that is
|
||||
* dirty.
|
||||
*
|
||||
* As an optimization, we could note that in practice we never allocate
|
||||
* out of a hugepage while purging within it, and so could try to
|
||||
* combine dirty extents separated by a non-dirty but non-active extent
|
||||
* to avoid purge calls. This does nontrivially complicate metadata
|
||||
* tracking though, so let's hold off for now.
|
||||
*/
|
||||
fb_bit_not(purge_state->to_purge, hpdata->active_pages, HUGEPAGE_PAGES);
|
||||
fb_bit_and(purge_state->to_purge, purge_state->to_purge,
|
||||
hpdata->dirty_pages, HUGEPAGE_PAGES);
|
||||
|
||||
/* We purge everything we can. */
|
||||
assert(hpdata->h_ndirty - hpdata->h_nactive == fb_scount(
|
||||
purge_state->to_purge, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES));
|
||||
|
||||
hpdata_assert_consistent(hpdata);
|
||||
}
|
||||
|
||||
bool
|
||||
hpdata_purge_next(hpdata_t *hpdata, hpdata_purge_state_t *purge_state,
|
||||
void **r_purge_addr, size_t *r_purge_size) {
|
||||
/*
|
||||
* Note that we don't have a consistency check here; we're accessing
|
||||
* hpdata without synchronization, and therefore have no right to expect
|
||||
* a consistent state.
|
||||
*/
|
||||
assert(hpdata->h_mid_purge);
|
||||
/* Should have dehugified already (if necessary). */
|
||||
assert(!hpdata->h_huge);
|
||||
assert(!hpdata->h_mid_hugify);
|
||||
|
||||
if (purge_state->next_purge_search_begin == HUGEPAGE_PAGES) {
|
||||
return false;
|
||||
}
|
||||
size_t purge_begin;
|
||||
size_t purge_len;
|
||||
bool found_range = fb_srange_iter(purge_state->to_purge, HUGEPAGE_PAGES,
|
||||
purge_state->next_purge_search_begin, &purge_begin, &purge_len);
|
||||
if (!found_range) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*r_purge_addr = (void *)(
|
||||
(uintptr_t)hpdata_addr_get(hpdata) + purge_begin * PAGE);
|
||||
*r_purge_size = purge_len * PAGE;
|
||||
|
||||
purge_state->next_purge_search_begin = purge_begin + purge_len;
|
||||
purge_state->npurged += purge_len;
|
||||
assert(purge_state->npurged <= HUGEPAGE_PAGES);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
hpdata_purge_end(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
|
||||
hpdata_assert_consistent(hpdata);
|
||||
assert(hpdata->h_mid_purge);
|
||||
assert(!hpdata->h_mid_hugify);
|
||||
hpdata->h_mid_purge = false;
|
||||
|
||||
assert(purge_state->npurged == fb_scount(purge_state->to_purge,
|
||||
HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES));
|
||||
|
||||
fb_bit_not(purge_state->to_purge, purge_state->to_purge,
|
||||
HUGEPAGE_PAGES);
|
||||
fb_bit_and(hpdata->dirty_pages, hpdata->dirty_pages,
|
||||
purge_state->to_purge, HUGEPAGE_PAGES);
|
||||
assert(hpdata->h_ndirty >= purge_state->npurged);
|
||||
hpdata->h_ndirty -= purge_state->npurged;
|
||||
|
||||
hpdata_assert_consistent(hpdata);
|
||||
}
|
||||
|
||||
void
|
||||
hpdata_hugify_begin(hpdata_t *hpdata) {
|
||||
hpdata_assert_consistent(hpdata);
|
||||
assert(!hpdata->h_mid_purge);
|
||||
assert(!hpdata->h_mid_hugify);
|
||||
hpdata->h_mid_hugify = true;
|
||||
hpdata->h_huge = true;
|
||||
fb_set_range(hpdata->dirty_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES);
|
||||
hpdata->h_ndirty = HUGEPAGE_PAGES;
|
||||
hpdata_assert_consistent(hpdata);
|
||||
}
|
||||
|
||||
void
|
||||
hpdata_hugify_end(hpdata_t *hpdata) {
|
||||
hpdata_assert_consistent(hpdata);
|
||||
assert(!hpdata->h_mid_purge);
|
||||
assert(hpdata->h_mid_hugify);
|
||||
hpdata->h_mid_hugify = false;
|
||||
hpdata_assert_consistent(hpdata);
|
||||
}
|
||||
|
||||
void
|
||||
hpdata_dehugify(hpdata_t *hpdata) {
|
||||
hpdata_assert_consistent(hpdata);
|
||||
/*
|
||||
* These asserts are morally right; for now, though, we have the "purge a
|
||||
* hugepage only in its entirety, when it becomes empty", path sharing
|
||||
* hpdata_dehugify with the new purge pathway coming in the next
|
||||
* commit.
|
||||
*/
|
||||
/*
|
||||
assert(hpdata->h_mid_purge);
|
||||
assert(!hpdata->h_mid_hugify);
|
||||
*/
|
||||
hpdata->h_huge = false;
|
||||
hpdata_assert_consistent(hpdata);
|
||||
}
|
||||
|
@ -55,7 +55,132 @@ TEST_BEGIN(test_reserve_alloc) {
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_purge_simple) {
|
||||
hpdata_t hpdata;
|
||||
hpdata_init(&hpdata, HPDATA_ADDR, HPDATA_AGE);
|
||||
|
||||
void *alloc = hpdata_reserve_alloc(&hpdata, HUGEPAGE_PAGES / 2 * PAGE);
|
||||
expect_ptr_eq(alloc, HPDATA_ADDR, "");
|
||||
|
||||
/* Create HUGEPAGE_PAGES / 4 dirty inactive pages at the beginning. */
|
||||
hpdata_unreserve(&hpdata, alloc, HUGEPAGE_PAGES / 4 * PAGE);
|
||||
|
||||
expect_zu_eq(hpdata_ndirty_get(&hpdata), HUGEPAGE_PAGES / 2, "");
|
||||
|
||||
expect_false(hpdata_changing_state_get(&hpdata), "");
|
||||
|
||||
hpdata_purge_state_t purge_state;
|
||||
hpdata_purge_begin(&hpdata, &purge_state);
|
||||
|
||||
expect_true(hpdata_changing_state_get(&hpdata), "");
|
||||
|
||||
void *purge_addr;
|
||||
size_t purge_size;
|
||||
bool got_result = hpdata_purge_next(&hpdata, &purge_state, &purge_addr,
|
||||
&purge_size);
|
||||
expect_true(got_result, "");
|
||||
expect_ptr_eq(HPDATA_ADDR, purge_addr, "");
|
||||
expect_zu_eq(HUGEPAGE_PAGES / 4 * PAGE, purge_size, "");
|
||||
|
||||
expect_true(hpdata_changing_state_get(&hpdata), "");
|
||||
|
||||
got_result = hpdata_purge_next(&hpdata, &purge_state, &purge_addr,
|
||||
&purge_size);
|
||||
expect_false(got_result, "Unexpected additional purge range: "
|
||||
"extent at %p of size %zu", purge_addr, purge_size);
|
||||
|
||||
expect_true(hpdata_changing_state_get(&hpdata), "");
|
||||
|
||||
hpdata_purge_end(&hpdata, &purge_state);
|
||||
expect_false(hpdata_changing_state_get(&hpdata), "");
|
||||
expect_zu_eq(hpdata_ndirty_get(&hpdata), HUGEPAGE_PAGES / 4, "");
|
||||
}
|
||||
TEST_END
|
||||
|
||||
/*
|
||||
* We only test intervening dalloc's not intervening allocs; we don't need
|
||||
* intervening allocs, and foreseeable optimizations will make them not just
|
||||
* unnecessary but incorrect. In particular, if there are two dirty extents
|
||||
* separated only by a retained extent, we can just purge the entire range,
|
||||
* saving a purge call.
|
||||
*/
|
||||
TEST_BEGIN(test_purge_intervening_dalloc) {
|
||||
hpdata_t hpdata;
|
||||
hpdata_init(&hpdata, HPDATA_ADDR, HPDATA_AGE);
|
||||
|
||||
/* Allocate the first 3/4 of the pages. */
|
||||
void *alloc = hpdata_reserve_alloc(&hpdata, 3 * HUGEPAGE_PAGES / 4 * PAGE);
|
||||
expect_ptr_eq(alloc, HPDATA_ADDR, "");
|
||||
|
||||
/* Free the first 1/4 and the third 1/4 of the pages. */
|
||||
hpdata_unreserve(&hpdata, alloc, HUGEPAGE_PAGES / 4 * PAGE);
|
||||
hpdata_unreserve(&hpdata,
|
||||
(void *)((uintptr_t)alloc + 2 * HUGEPAGE_PAGES / 4 * PAGE),
|
||||
HUGEPAGE_PAGES / 4 * PAGE);
|
||||
|
||||
expect_zu_eq(hpdata_ndirty_get(&hpdata), 3 * HUGEPAGE_PAGES / 4, "");
|
||||
|
||||
hpdata_purge_state_t purge_state;
|
||||
hpdata_purge_begin(&hpdata, &purge_state);
|
||||
|
||||
void *purge_addr;
|
||||
size_t purge_size;
|
||||
/* First purge. */
|
||||
bool got_result = hpdata_purge_next(&hpdata, &purge_state, &purge_addr,
|
||||
&purge_size);
|
||||
expect_true(got_result, "");
|
||||
expect_ptr_eq(HPDATA_ADDR, purge_addr, "");
|
||||
expect_zu_eq(HUGEPAGE_PAGES / 4 * PAGE, purge_size, "");
|
||||
|
||||
/* Deallocate the second 1/4 before the second purge occurs. */
|
||||
hpdata_unreserve(&hpdata,
|
||||
(void *)((uintptr_t)alloc + 1 * HUGEPAGE_PAGES / 4 * PAGE),
|
||||
HUGEPAGE_PAGES / 4 * PAGE);
|
||||
|
||||
/* Now continue purging. */
|
||||
got_result = hpdata_purge_next(&hpdata, &purge_state, &purge_addr,
|
||||
&purge_size);
|
||||
expect_true(got_result, "");
|
||||
expect_ptr_eq(
|
||||
(void *)((uintptr_t)alloc + 2 * HUGEPAGE_PAGES / 4 * PAGE),
|
||||
purge_addr, "");
|
||||
expect_zu_eq(HUGEPAGE_PAGES / 4 * PAGE, purge_size, "");
|
||||
|
||||
got_result = hpdata_purge_next(&hpdata, &purge_state, &purge_addr,
|
||||
&purge_size);
|
||||
expect_false(got_result, "Unexpected additional purge range: "
|
||||
"extent at %p of size %zu", purge_addr, purge_size);
|
||||
|
||||
hpdata_purge_end(&hpdata, &purge_state);
|
||||
|
||||
expect_zu_eq(hpdata_ndirty_get(&hpdata), HUGEPAGE_PAGES / 4, "");
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_hugify) {
|
||||
hpdata_t hpdata;
|
||||
hpdata_init(&hpdata, HPDATA_ADDR, HPDATA_AGE);
|
||||
|
||||
void *alloc = hpdata_reserve_alloc(&hpdata, HUGEPAGE / 2);
|
||||
expect_ptr_eq(alloc, HPDATA_ADDR, "");
|
||||
|
||||
expect_zu_eq(HUGEPAGE_PAGES / 2, hpdata_ndirty_get(&hpdata), "");
|
||||
|
||||
expect_false(hpdata_changing_state_get(&hpdata), "");
|
||||
hpdata_hugify_begin(&hpdata);
|
||||
expect_true(hpdata_changing_state_get(&hpdata), "");
|
||||
hpdata_hugify_end(&hpdata);
|
||||
expect_false(hpdata_changing_state_get(&hpdata), "");
|
||||
|
||||
/* Hugeifying should have increased the dirty page count. */
|
||||
expect_zu_eq(HUGEPAGE_PAGES, hpdata_ndirty_get(&hpdata), "");
|
||||
}
|
||||
TEST_END
|
||||
|
||||
int main(void) {
|
||||
return test_no_reentrancy(
|
||||
test_reserve_alloc);
|
||||
test_reserve_alloc,
|
||||
test_purge_simple,
|
||||
test_purge_intervening_dalloc,
|
||||
test_hugify);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user