psset: Use fit/insert/remove as basis functions.
All other functionality can be implemented in terms of these; doing so (while retaining the same API) will be convenient for subsequent refactors.
This commit is contained in:
parent
089f8fa442
commit
5228d869ee
@ -123,6 +123,11 @@ TYPED_LIST(hpdata_list, hpdata_t, ql_link)
|
||||
typedef ph(hpdata_t) hpdata_age_heap_t;
|
||||
ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t);
|
||||
|
||||
static inline bool
|
||||
hpdata_empty(hpdata_t *hpdata) {
|
||||
return hpdata_nfree_get(hpdata) == HUGEPAGE_PAGES;
|
||||
}
|
||||
|
||||
void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
|
||||
/*
|
||||
* Given an hpdata which can serve an allocation request, pick and reserve an
|
||||
|
@ -64,13 +64,8 @@ void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src);
|
||||
void psset_insert(psset_t *psset, hpdata_t *ps);
|
||||
void psset_remove(psset_t *psset, hpdata_t *ps);
|
||||
|
||||
void psset_hugify(psset_t *psset, hpdata_t *ps);
|
||||
|
||||
/*
|
||||
* Tries to obtain a chunk from an existing pageslab already in the set.
|
||||
* Returns true on failure.
|
||||
*/
|
||||
bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
|
||||
/* Analogous to the eset_fit; pick a hpdata to serve the request. */
|
||||
hpdata_t *psset_fit(psset_t *psset, size_t size);
|
||||
|
||||
/*
|
||||
* Given a newly created pageslab ps (not currently in the set), pass ownership
|
||||
@ -79,6 +74,7 @@ bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
|
||||
*/
|
||||
void psset_alloc_new(psset_t *psset, hpdata_t *ps,
|
||||
edata_t *r_edata, size_t size);
|
||||
bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
|
||||
|
||||
/*
|
||||
* Given an extent that comes from a pageslab in this pageslab set, returns it
|
||||
|
12
src/hpa.c
12
src/hpa.c
@ -129,7 +129,7 @@ hpa_hugify(hpdata_t *ps) {
|
||||
assert(hpdata_huge_get(ps));
|
||||
bool err = pages_huge(hpdata_addr_get(ps), HUGEPAGE);
|
||||
/*
|
||||
* Eat the error; even if the hugeification failed, it's still safe to
|
||||
* Eat the error; even if the hugification failed, it's still safe to
|
||||
* pretend it didn't (and would require extraordinary measures to
|
||||
* unhugify).
|
||||
*/
|
||||
@ -233,7 +233,7 @@ hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, hpdata_t *ps) {
|
||||
|
||||
/*
|
||||
* We do this unconditionally, even for pages which were not originally
|
||||
* hugeified; it has the same effect.
|
||||
* hugified; it has the same effect.
|
||||
*/
|
||||
hpa_dehugify(ps);
|
||||
|
||||
@ -293,7 +293,9 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
|
||||
* Do the metadata modification while holding the lock; we'll
|
||||
* actually change state with the lock dropped.
|
||||
*/
|
||||
psset_hugify(&shard->psset, ps);
|
||||
psset_remove(&shard->psset, ps);
|
||||
hpdata_huge_set(ps, true);
|
||||
psset_insert(&shard->psset, ps);
|
||||
}
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
if (hugify) {
|
||||
@ -463,8 +465,8 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
|
||||
emap_deregister_boundary(tsdn, shard->emap, edata);
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
/*
|
||||
* Note that the shard mutex protects the edata hugeified field, too.
|
||||
* Page slabs can move between pssets (and have their hugeified status
|
||||
* Note that the shard mutex protects the edata hugified field, too.
|
||||
* Page slabs can move between pssets (and have their hugified status
|
||||
* change) in racy ways.
|
||||
*/
|
||||
hpdata_t *evicted_ps = psset_dalloc(&shard->psset, edata);
|
||||
|
160
src/psset.c
160
src/psset.c
@ -76,23 +76,6 @@ psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) {
|
||||
psset_bin_stats_insert_remove(binstats, ps, /* insert */ false);
|
||||
}
|
||||
|
||||
/*
|
||||
* We don't currently need an "activate" equivalent to this, since down the
|
||||
* allocation pathways we don't do the optimization in which we change a slab
|
||||
* without first removing it from a bin.
|
||||
*/
|
||||
static void
|
||||
psset_bin_stats_deactivate(psset_bin_stats_t *binstats, bool huge, size_t num) {
|
||||
size_t *nactive_dst = huge
|
||||
? &binstats->nactive_huge : &binstats->nactive_nonhuge;
|
||||
size_t *ninactive_dst = huge
|
||||
? &binstats->ninactive_huge : &binstats->ninactive_nonhuge;
|
||||
|
||||
assert(*nactive_dst >= num);
|
||||
*nactive_dst -= num;
|
||||
*ninactive_dst += num;
|
||||
}
|
||||
|
||||
static void
|
||||
psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
|
||||
hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
|
||||
@ -148,32 +131,8 @@ psset_remove(psset_t *psset, hpdata_t *ps) {
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
psset_hugify(psset_t *psset, hpdata_t *ps) {
|
||||
assert(!hpdata_huge_get(ps));
|
||||
hpdata_assert_consistent(ps);
|
||||
|
||||
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
||||
psset_bin_stats_t *bin_stats;
|
||||
if (longest_free_range == 0) {
|
||||
bin_stats = &psset->stats.full_slabs;
|
||||
} else {
|
||||
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
|
||||
longest_free_range << LG_PAGE));
|
||||
assert(pind < PSSET_NPSIZES);
|
||||
bin_stats = &psset->stats.nonfull_slabs[pind];
|
||||
}
|
||||
psset_bin_stats_remove(bin_stats, ps);
|
||||
hpdata_huge_set(ps, true);
|
||||
psset_bin_stats_insert(bin_stats, ps);
|
||||
}
|
||||
|
||||
/*
|
||||
* Similar to PAC's extent_recycle_extract. Out of all the pageslabs in the
|
||||
* set, picks one that can satisfy the allocation and remove it from the set.
|
||||
*/
|
||||
static hpdata_t *
|
||||
psset_recycle_extract(psset_t *psset, size_t size) {
|
||||
hpdata_t *
|
||||
psset_fit(psset_t *psset, size_t size) {
|
||||
pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
|
||||
pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
|
||||
(size_t)min_pind);
|
||||
@ -185,22 +144,14 @@ psset_recycle_extract(psset_t *psset, size_t size) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
psset_hpdata_heap_remove(psset, pind, ps);
|
||||
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
|
||||
bitmap_set(psset->bitmap, &psset_bitmap_info, pind);
|
||||
}
|
||||
|
||||
hpdata_assert_consistent(ps);
|
||||
|
||||
return ps;
|
||||
}
|
||||
|
||||
/*
|
||||
* Given a pageslab ps and an edata to allocate size bytes from, initializes the
|
||||
* edata with a range in the pageslab, and puts ps back in the set.
|
||||
*/
|
||||
static void
|
||||
psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
|
||||
size_t size) {
|
||||
void
|
||||
psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
|
||||
hpdata_assert_empty(ps);
|
||||
size_t npages = size / PAGE;
|
||||
size_t begin = hpdata_reserve_alloc(ps, npages);
|
||||
uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
|
||||
@ -209,30 +160,28 @@ psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
|
||||
/* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
|
||||
EXTENT_NOT_HEAD);
|
||||
edata_ps_set(r_edata, ps);
|
||||
/* The pageslab isn't in a bin, so no bin stats need to change. */
|
||||
|
||||
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
||||
if (longest_free_range == 0) {
|
||||
psset_bin_stats_insert(&psset->stats.full_slabs, ps);
|
||||
} else {
|
||||
psset_insert(psset, ps);
|
||||
}
|
||||
psset_insert(psset, ps);
|
||||
}
|
||||
|
||||
bool
|
||||
psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) {
|
||||
hpdata_t *ps = psset_recycle_extract(psset, size);
|
||||
if (ps == NULL) {
|
||||
return true;
|
||||
}
|
||||
psset_ps_alloc_insert(psset, ps, r_edata, size);
|
||||
return false;
|
||||
}
|
||||
hpdata_t *ps = psset_fit(psset, size);
|
||||
if (ps == NULL) {
|
||||
return true;
|
||||
}
|
||||
psset_remove(psset, ps);
|
||||
|
||||
void
|
||||
psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
|
||||
hpdata_assert_empty(ps);
|
||||
psset_ps_alloc_insert(psset, ps, r_edata, size);
|
||||
size_t npages = size / PAGE;
|
||||
size_t begin = hpdata_reserve_alloc(ps, npages);
|
||||
uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
|
||||
edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size,
|
||||
/* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active,
|
||||
/* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
|
||||
EXTENT_NOT_HEAD);
|
||||
edata_ps_set(r_edata, ps);
|
||||
psset_insert(psset, ps);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
hpdata_t *
|
||||
@ -241,70 +190,17 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
|
||||
assert(edata_ps_get(edata) != NULL);
|
||||
hpdata_t *ps = edata_ps_get(edata);
|
||||
|
||||
size_t ps_old_longest_free_range = hpdata_longest_free_range_get(ps);
|
||||
pszind_t old_pind = SC_NPSIZES;
|
||||
if (ps_old_longest_free_range != 0) {
|
||||
old_pind = sz_psz2ind(sz_psz_quantize_floor(
|
||||
ps_old_longest_free_range << LG_PAGE));
|
||||
}
|
||||
|
||||
size_t begin =
|
||||
((uintptr_t)edata_base_get(edata) - (uintptr_t)hpdata_addr_get(ps))
|
||||
>> LG_PAGE;
|
||||
size_t len = edata_size_get(edata) >> LG_PAGE;
|
||||
|
||||
/* The pageslab is still in the bin; adjust its stats first. */
|
||||
psset_bin_stats_t *bin_stats = (ps_old_longest_free_range == 0
|
||||
? &psset->stats.full_slabs : &psset->stats.nonfull_slabs[old_pind]);
|
||||
psset_bin_stats_deactivate(bin_stats, hpdata_huge_get(ps), len);
|
||||
|
||||
psset_remove(psset, ps);
|
||||
hpdata_unreserve(ps, begin, len);
|
||||
size_t ps_new_longest_free_range = hpdata_longest_free_range_get(ps);
|
||||
|
||||
/*
|
||||
* If the new free range is no longer than the previous longest one,
|
||||
* then the pageslab is non-empty and doesn't need to change bins.
|
||||
* We're done, and don't need to return a pageslab to evict.
|
||||
*/
|
||||
if (ps_new_longest_free_range <= ps_old_longest_free_range) {
|
||||
if (hpdata_empty(ps)) {
|
||||
return ps;
|
||||
} else {
|
||||
psset_insert(psset, ps);
|
||||
return NULL;
|
||||
}
|
||||
/*
|
||||
* If it was previously non-full, then it's in some (possibly now
|
||||
* incorrect) bin already; remove it.
|
||||
*
|
||||
* TODO: We bailed out early above if we didn't expand the longest free
|
||||
* range, which should avoid a lot of redundant remove/reinserts in the
|
||||
* same bin. But it doesn't eliminate all of them; it's possible that
|
||||
* we decreased the longest free range length, but only slightly, and
|
||||
* not enough to change our pszind. We could check that more precisely.
|
||||
* (Or, ideally, size class dequantization will happen at some point,
|
||||
* and the issue becomes moot).
|
||||
*/
|
||||
if (ps_old_longest_free_range > 0) {
|
||||
psset_hpdata_heap_remove(psset, old_pind, ps);
|
||||
if (hpdata_age_heap_empty(&psset->pageslabs[old_pind])) {
|
||||
bitmap_set(psset->bitmap, &psset_bitmap_info,
|
||||
(size_t)old_pind);
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
* Otherwise, the bin was full, and we need to adjust the full
|
||||
* bin stats.
|
||||
*/
|
||||
psset_bin_stats_remove(&psset->stats.full_slabs, ps);
|
||||
}
|
||||
/* If the pageslab is empty, it gets evicted from the set. */
|
||||
if (ps_new_longest_free_range == HUGEPAGE_PAGES) {
|
||||
return ps;
|
||||
}
|
||||
/* Otherwise, it gets reinserted. */
|
||||
pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor(
|
||||
ps_new_longest_free_range << LG_PAGE));
|
||||
if (hpdata_age_heap_empty(&psset->pageslabs[new_pind])) {
|
||||
bitmap_unset(psset->bitmap, &psset_bitmap_info,
|
||||
(size_t)new_pind);
|
||||
}
|
||||
psset_hpdata_heap_insert(psset, new_pind, ps);
|
||||
return NULL;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user