project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

psset: Reconceptualize insertion/removal.

Browse Source 
Really, this isn't a functional change, just a naming change.  We start thinking
of pageslabs as being always in the psset.  What we used to think of as removal
is now thought of as being in the psset, but in the process of being updated
(and therefore, unavalable for serving new allocations).

This is in preparation of subsequent changes to support deferred purging;
allocations will still be in the psset for the purposes of choosing when to
purge, but not for purposes of allocation/deallocation.
This commit is contained in:
David Goldblatt
2020-12-05 15:58:31 -08:00
committed by David Goldblatt
parent 061cabb712
commit 99fc0717e6
7 changed files with 106 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/hpa.c
View File

@@ -333,14 +333,14 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
}
assert(edata_arena_ind_get(edata) == shard->ind);
hpdata_t *ps = psset_fit(&shard->psset, size);
hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
if (ps == NULL) {
edata_cache_small_put(tsdn, &shard->ecs, edata);
malloc_mutex_unlock(tsdn, &shard->mtx);
return NULL;
}
psset_remove(&shard->psset, ps);
psset_update_begin(&shard->psset, ps);
void *addr = hpdata_reserve_alloc(ps, size);
edata_init(edata, shard->ind, addr, size, /* slab */ false,
SC_NSIZES, /* sn */ 0, extent_state_active, /* zeroed */ false,
@@ -365,7 +365,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
* require some sort of prepare + commit functionality that's a
* little much to deal with for now.
*/
psset_insert(&shard->psset, ps);
psset_update_end(&shard->psset, ps);
edata_cache_small_put(tsdn, &shard->ecs, edata);
malloc_mutex_unlock(tsdn, &shard->mtx);
*oom = true;
@@ -377,7 +377,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
hpdata_hugify_begin(ps);
shard->stats.nhugifies++;
}
psset_insert(&shard->psset, ps);
psset_update_end(&shard->psset, ps);
malloc_mutex_unlock(tsdn, &shard->mtx);
if (hugify) {
@@ -409,9 +409,9 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
* hugified. Undo our operation, taking care to meet
* the precondition that the ps isn't in the psset.
*/
psset_remove(&shard->psset, ps);
psset_update_begin(&shard->psset, ps);
hpa_purge(tsdn, shard, ps);
psset_insert(&shard->psset, ps);
psset_update_end(&shard->psset, ps);
}
malloc_mutex_unlock(tsdn, &shard->mtx);
}
@@ -455,6 +455,15 @@ hpa_alloc_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size) {
/* We got the new edata; allocate from it. */
malloc_mutex_lock(tsdn, &shard->mtx);
/*
* This will go away soon. The psset doesn't draw a distinction between
* pageslab removal and updating. If this is a new pageslab, we pretend
* that it's an old one that's been getting updated.
*/
if (!hpdata_updating_get(ps)) {
hpdata_updating_set(ps, true);
}
edata = edata_cache_small_get(tsdn, &shard->ecs);
if (edata == NULL) {
shard->stats.nevictions++;
@@ -500,7 +509,7 @@ hpa_alloc_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size) {
hpa_handle_ps_eviction(tsdn, shard, ps);
return NULL;
}
psset_insert(&shard->psset, ps);
psset_update_end(&shard->psset, ps);
malloc_mutex_unlock(tsdn, &shard->mtx);
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
@@ -615,7 +624,7 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
* psset and we can do our metadata update. The other thread is
* in charge of reinserting the ps, so we're done.
*/
assert(!hpdata_in_psset_get(ps));
assert(hpdata_updating_get(ps));
hpdata_unreserve(ps, unreserve_addr, unreserve_size);
malloc_mutex_unlock(tsdn, &shard->mtx);
return;
@@ -624,15 +633,15 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
* No other thread is purging, and the ps is non-empty, so it should be
* in the psset.
*/
assert(hpdata_in_psset_get(ps));
psset_remove(&shard->psset, ps);
assert(!hpdata_updating_get(ps));
psset_update_begin(&shard->psset, ps);
hpdata_unreserve(ps, unreserve_addr, unreserve_size);
if (!hpa_should_purge(shard, ps)) {
/*
* This should be the common case; no other thread is purging,
* and we won't purge either.
*/
psset_insert(&shard->psset, ps);
psset_update_end(&shard->psset, ps);
malloc_mutex_unlock(tsdn, &shard->mtx);
return;
}
@@ -648,7 +657,7 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
malloc_mutex_unlock(tsdn, &shard->mtx);
hpa_handle_ps_eviction(tsdn, shard, ps);
} else {
psset_insert(&shard->psset, ps);
psset_update_end(&shard->psset, ps);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
}
@@ -669,7 +678,7 @@ hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) {
static void
hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
hpdata_t *ps = psset_fit(psset, PAGE);
hpdata_t *ps = psset_pick_alloc(psset, PAGE);
assert(ps == NULL);
for (int huge = 0; huge <= 1; huge++) {
hpa_shard_assert_stats_empty(&psset->stats.full_slabs[huge]);

23
src/hpdata.c
View File

@@ -24,7 +24,7 @@ hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
hpdata->h_huge = false;
hpdata->h_mid_purge = false;
hpdata->h_mid_hugify = false;
hpdata->h_in_psset = false;
hpdata->h_updating = false;
hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES);
hpdata->h_nactive = 0;
fb_init(hpdata->active_pages, HUGEPAGE_PAGES);
@@ -37,7 +37,7 @@ hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
void *
hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
hpdata_assert_consistent(hpdata);
assert(!hpdata_in_psset_get(hpdata));
assert(hpdata->h_updating);
assert((sz & PAGE_MASK) == 0);
size_t npages = sz >> LG_PAGE;
assert(npages <= hpdata_longest_free_range_get(hpdata));
@@ -118,7 +118,7 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
void
hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
hpdata_assert_consistent(hpdata);
assert(!hpdata->h_in_psset);
assert(hpdata->h_updating);
assert(((uintptr_t)addr & PAGE_MASK) == 0);
assert((sz & PAGE_MASK) == 0);
size_t begin = ((uintptr_t)addr - (uintptr_t)hpdata_addr_get(hpdata))
@@ -147,7 +147,7 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
void
hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
hpdata_assert_consistent(hpdata);
assert(!hpdata->h_in_psset);
assert(hpdata->h_updating);
assert(!hpdata->h_mid_purge);
assert(!hpdata->h_mid_hugify);
hpdata->h_mid_purge = true;
@@ -185,7 +185,7 @@ hpdata_purge_next(hpdata_t *hpdata, hpdata_purge_state_t *purge_state,
* a consistent state.
*/
assert(hpdata->h_mid_purge);
assert(!hpdata->h_in_psset);
assert(hpdata->h_updating);
/* Should have dehugified already (if necessary). */
assert(!hpdata->h_huge);
assert(!hpdata->h_mid_hugify);
@@ -215,7 +215,7 @@ hpdata_purge_next(hpdata_t *hpdata, hpdata_purge_state_t *purge_state,
void
hpdata_purge_end(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
hpdata_assert_consistent(hpdata);
assert(!hpdata->h_in_psset);
assert(hpdata->h_updating);
assert(hpdata->h_mid_purge);
assert(!hpdata->h_mid_hugify);
hpdata->h_mid_purge = false;
@@ -236,7 +236,7 @@ hpdata_purge_end(hpdata_t *hpdata, hpdata_purge_state_t *purge_state) {
void
hpdata_hugify_begin(hpdata_t *hpdata) {
hpdata_assert_consistent(hpdata);
assert(!hpdata_in_psset_get(hpdata));
assert(hpdata->h_updating);
assert(!hpdata->h_mid_purge);
assert(!hpdata->h_mid_hugify);
hpdata->h_mid_hugify = true;
@@ -250,10 +250,10 @@ void
hpdata_hugify_end(hpdata_t *hpdata) {
hpdata_assert_consistent(hpdata);
/*
* This is the exception to the "no metadata tweaks while in the psset"
* rule.
* This is the exception to the "no-metadata updates without informing
* the psset first" rule; this assert would be incorrect.
*/
/* assert(!hpdata_in_psset_get(hpdata)); */
/* assert(hpdata->h_updating); */
assert(!hpdata->h_mid_purge);
assert(hpdata->h_mid_hugify);
hpdata->h_mid_hugify = false;
@@ -263,7 +263,8 @@ hpdata_hugify_end(hpdata_t *hpdata) {
void
hpdata_dehugify(hpdata_t *hpdata) {
hpdata_assert_consistent(hpdata);
assert(!hpdata_in_psset_get(hpdata));
assert(hpdata->h_updating);
assert(hpdata->h_updating);
assert(hpdata->h_mid_purge);
assert(!hpdata->h_mid_hugify);
hpdata->h_huge = false;

52
src/psset.c
View File

@@ -79,11 +79,33 @@ psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
}
void
psset_insert(psset_t *psset, hpdata_t *ps) {
psset_update_begin(psset_t *psset, hpdata_t *ps) {
hpdata_assert_consistent(ps);
assert(!hpdata_updating_get(ps));
hpdata_updating_set(ps, true);
size_t longest_free_range = hpdata_longest_free_range_get(ps);
if (longest_free_range == 0) {
psset_bin_stats_remove(psset->stats.full_slabs, ps);
return;
}
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
longest_free_range << LG_PAGE));
assert(pind < PSSET_NPSIZES);
psset_hpdata_heap_remove(psset, pind, ps);
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind);
}
}
void
psset_update_end(psset_t *psset, hpdata_t *ps) {
assert(!hpdata_empty(ps));
hpdata_assert_consistent(ps);
assert(!hpdata_in_psset_get(ps));
hpdata_in_psset_set(ps, true);
assert(hpdata_updating_get(ps));
hpdata_updating_set(ps, false);
size_t longest_free_range = hpdata_longest_free_range_get(ps);
if (longest_free_range == 0) {
@@ -105,30 +127,8 @@ psset_insert(psset_t *psset, hpdata_t *ps) {
psset_hpdata_heap_insert(psset, pind, ps);
}
void
psset_remove(psset_t *psset, hpdata_t *ps) {
hpdata_assert_consistent(ps);
assert(hpdata_in_psset_get(ps));
hpdata_in_psset_set(ps, false);
size_t longest_free_range = hpdata_longest_free_range_get(ps);
if (longest_free_range == 0) {
psset_bin_stats_remove(psset->stats.full_slabs, ps);
return;
}
pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
longest_free_range << LG_PAGE));
assert(pind < PSSET_NPSIZES);
psset_hpdata_heap_remove(psset, pind, ps);
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind);
}
}
hpdata_t *
psset_fit(psset_t *psset, size_t size) {
psset_pick_alloc(psset_t *psset, size_t size) {
pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
(size_t)min_pind);