project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

psset: Use fit/insert/remove as basis functions.

Browse Source 
All other functionality can be implemented in terms of these; doing so (while
retaining the same API) will be convenient for subsequent refactors.
This commit is contained in:
David Goldblatt 2020-11-30 13:28:54 -08:00 committed by David Goldblatt
parent 089f8fa442
commit 5228d869ee
4 changed files with 43 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/jemalloc/internal/hpdata.h
View File

@ -123,6 +123,11 @@ TYPED_LIST(hpdata_list, hpdata_t, ql_link)
typedef ph(hpdata_t) hpdata_age_heap_t;
ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t);
static inline bool
hpdata_empty(hpdata_t *hpdata) {
return hpdata_nfree_get(hpdata) == HUGEPAGE_PAGES;
}
void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
/*
* Given an hpdata which can serve an allocation request, pick and reserve an

10
include/jemalloc/internal/psset.h
View File

@ -64,13 +64,8 @@ void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src);
void psset_insert(psset_t *psset, hpdata_t *ps);
void psset_remove(psset_t *psset, hpdata_t *ps);
void psset_hugify(psset_t *psset, hpdata_t *ps);
/*
* Tries to obtain a chunk from an existing pageslab already in the set.
* Returns true on failure.
*/
bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
/* Analogous to the eset_fit; pick a hpdata to serve the request. */
hpdata_t *psset_fit(psset_t *psset, size_t size);
/*
* Given a newly created pageslab ps (not currently in the set), pass ownership
@ -79,6 +74,7 @@ bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
*/
void psset_alloc_new(psset_t *psset, hpdata_t *ps,
edata_t *r_edata, size_t size);
bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
/*
* Given an extent that comes from a pageslab in this pageslab set, returns it

12
src/hpa.c
View File

@ -129,7 +129,7 @@ hpa_hugify(hpdata_t *ps) {
assert(hpdata_huge_get(ps));
bool err = pages_huge(hpdata_addr_get(ps), HUGEPAGE);
/*
* Eat the error; even if the hugeification failed, it's still safe to
* Eat the error; even if the hugification failed, it's still safe to
* pretend it didn't (and would require extraordinary measures to
* unhugify).
*/
@ -233,7 +233,7 @@ hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, hpdata_t *ps) {
/*
* We do this unconditionally, even for pages which were not originally
* hugeified; it has the same effect.
* hugified; it has the same effect.
*/
hpa_dehugify(ps);
@ -293,7 +293,9 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
* Do the metadata modification while holding the lock; we'll
* actually change state with the lock dropped.
*/
psset_hugify(&shard->psset, ps);
psset_remove(&shard->psset, ps);
hpdata_huge_set(ps, true);
psset_insert(&shard->psset, ps);
}
malloc_mutex_unlock(tsdn, &shard->mtx);
if (hugify) {
@ -463,8 +465,8 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
emap_deregister_boundary(tsdn, shard->emap, edata);
malloc_mutex_lock(tsdn, &shard->mtx);
/*
* Note that the shard mutex protects the edata hugeified field, too.
* Page slabs can move between pssets (and have their hugeified status
* Note that the shard mutex protects the edata hugified field, too.
* Page slabs can move between pssets (and have their hugified status
* change) in racy ways.
*/
hpdata_t *evicted_ps = psset_dalloc(&shard->psset, edata);

152
src/psset.c
View File

@ -76,23 +76,6 @@ psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) {
psset_bin_stats_insert_remove(binstats, ps, /* insert */ false);
}
/*
* We don't currently need an "activate" equivalent to this, since down the
* allocation pathways we don't do the optimization in which we change a slab
* without first removing it from a bin.
*/
static void
psset_bin_stats_deactivate(psset_bin_stats_t *binstats, bool huge, size_t num) {
size_t *nactive_dst = huge
? &binstats->nactive_huge : &binstats->nactive_nonhuge;
size_t *ninactive_dst = huge
? &binstats->ninactive_huge : &binstats->ninactive_nonhuge;
assert(*nactive_dst >= num);
*nactive_dst -= num;
*ninactive_dst += num;
}
static void
psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
@ -148,32 +131,8 @@ psset_remove(psset_t *psset, hpdata_t *ps) {
}
}
void
psset_hugify(psset_t *psset, hpdata_t *ps) {
assert(!hpdata_huge_get(ps));
hpdata_assert_consistent(ps);
size_t longest_free_range = hpdata_longest_free_range_get(ps);
psset_bin_stats_t *bin_stats;
if (longest_free_range == 0) {
bin_stats = &psset->stats.full_slabs;
} else {
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
longest_free_range << LG_PAGE));
assert(pind < PSSET_NPSIZES);
bin_stats = &psset->stats.nonfull_slabs[pind];
}
psset_bin_stats_remove(bin_stats, ps);
hpdata_huge_set(ps, true);
psset_bin_stats_insert(bin_stats, ps);
}
/*
* Similar to PAC's extent_recycle_extract. Out of all the pageslabs in the
* set, picks one that can satisfy the allocation and remove it from the set.
*/
static hpdata_t *
psset_recycle_extract(psset_t *psset, size_t size) {
hpdata_t *
psset_fit(psset_t *psset, size_t size) {
pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
(size_t)min_pind);
@ -185,22 +144,14 @@ psset_recycle_extract(psset_t *psset, size_t size) {
return NULL;
}
psset_hpdata_heap_remove(psset, pind, ps);
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info, pind);
}
hpdata_assert_consistent(ps);
return ps;
}
/*
* Given a pageslab ps and an edata to allocate size bytes from, initializes the
* edata with a range in the pageslab, and puts ps back in the set.
*/
static void
psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
size_t size) {
void
psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
hpdata_assert_empty(ps);
size_t npages = size / PAGE;
size_t begin = hpdata_reserve_alloc(ps, npages);
uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
@ -209,30 +160,28 @@ psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
/* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
EXTENT_NOT_HEAD);
edata_ps_set(r_edata, ps);
/* The pageslab isn't in a bin, so no bin stats need to change. */
size_t longest_free_range = hpdata_longest_free_range_get(ps);
if (longest_free_range == 0) {
psset_bin_stats_insert(&psset->stats.full_slabs, ps);
} else {
psset_insert(psset, ps);
}
}
bool
psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) {
hpdata_t *ps = psset_recycle_extract(psset, size);
hpdata_t *ps = psset_fit(psset, size);
if (ps == NULL) {
return true;
}
psset_ps_alloc_insert(psset, ps, r_edata, size);
return false;
}
psset_remove(psset, ps);
void
psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
hpdata_assert_empty(ps);
psset_ps_alloc_insert(psset, ps, r_edata, size);
size_t npages = size / PAGE;
size_t begin = hpdata_reserve_alloc(ps, npages);
uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size,
/* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active,
/* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
EXTENT_NOT_HEAD);
edata_ps_set(r_edata, ps);
psset_insert(psset, ps);
return false;
}
hpdata_t *
@ -241,70 +190,17 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
assert(edata_ps_get(edata) != NULL);
hpdata_t *ps = edata_ps_get(edata);
size_t ps_old_longest_free_range = hpdata_longest_free_range_get(ps);
pszind_t old_pind = SC_NPSIZES;
if (ps_old_longest_free_range != 0) {
old_pind = sz_psz2ind(sz_psz_quantize_floor(
ps_old_longest_free_range << LG_PAGE));
}
size_t begin =
((uintptr_t)edata_base_get(edata) - (uintptr_t)hpdata_addr_get(ps))
>> LG_PAGE;
size_t len = edata_size_get(edata) >> LG_PAGE;
/* The pageslab is still in the bin; adjust its stats first. */
psset_bin_stats_t *bin_stats = (ps_old_longest_free_range == 0
? &psset->stats.full_slabs : &psset->stats.nonfull_slabs[old_pind]);
psset_bin_stats_deactivate(bin_stats, hpdata_huge_get(ps), len);
psset_remove(psset, ps);
hpdata_unreserve(ps, begin, len);
size_t ps_new_longest_free_range = hpdata_longest_free_range_get(ps);
/*
* If the new free range is no longer than the previous longest one,
* then the pageslab is non-empty and doesn't need to change bins.
* We're done, and don't need to return a pageslab to evict.
*/
if (ps_new_longest_free_range <= ps_old_longest_free_range) {
return NULL;
}
/*
* If it was previously non-full, then it's in some (possibly now
* incorrect) bin already; remove it.
*
* TODO: We bailed out early above if we didn't expand the longest free
* range, which should avoid a lot of redundant remove/reinserts in the
* same bin. But it doesn't eliminate all of them; it's possible that
* we decreased the longest free range length, but only slightly, and
* not enough to change our pszind. We could check that more precisely.
* (Or, ideally, size class dequantization will happen at some point,
* and the issue becomes moot).
*/
if (ps_old_longest_free_range > 0) {
psset_hpdata_heap_remove(psset, old_pind, ps);
if (hpdata_age_heap_empty(&psset->pageslabs[old_pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info,
(size_t)old_pind);
}
} else {
/*
* Otherwise, the bin was full, and we need to adjust the full
* bin stats.
*/
psset_bin_stats_remove(&psset->stats.full_slabs, ps);
}
/* If the pageslab is empty, it gets evicted from the set. */
if (ps_new_longest_free_range == HUGEPAGE_PAGES) {
if (hpdata_empty(ps)) {
return ps;
}
/* Otherwise, it gets reinserted. */
pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor(
ps_new_longest_free_range << LG_PAGE));
if (hpdata_age_heap_empty(&psset->pageslabs[new_pind])) {
bitmap_unset(psset->bitmap, &psset_bitmap_info,
(size_t)new_pind);
}
psset_hpdata_heap_insert(psset, new_pind, ps);
} else {
psset_insert(psset, ps);
return NULL;
}
}