project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extent refactor: Introduce ecache module.

Browse Source 
This will eventually completely wrap the eset, and handle concurrency,
allocation, and deallocation.  For now, we only pull out the mutex from the
eset.
This commit is contained in:
David Goldblatt
2019-12-12 16:25:24 -08:00
committed by David Goldblatt
parent 0704516245
commit bb70df8e5b
15 changed files with 354 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
src/arena.c
View File

@@ -56,7 +56,7 @@ static unsigned huge_arena_ind;
*/
static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena,
arena_decay_t *decay, eset_t *eset, bool all, size_t npages_limit,
arena_decay_t *decay, ecache_t *ecache, bool all, size_t npages_limit,
size_t npages_decay_max, bool is_background_thread);
static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena,
bool is_background_thread, bool all);
@@ -76,8 +76,8 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
*dirty_decay_ms = arena_dirty_decay_ms_get(arena);
*muzzy_decay_ms = arena_muzzy_decay_ms_get(arena);
*nactive += atomic_load_zu(&arena->nactive, ATOMIC_RELAXED);
*ndirty += eset_npages_get(&arena->eset_dirty);
*nmuzzy += eset_npages_get(&arena->eset_muzzy);
*ndirty += ecache_npages_get(&arena->ecache_dirty);
*nmuzzy += ecache_npages_get(&arena->ecache_muzzy);
}
void
@@ -100,7 +100,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_stats_accum_zu(&astats->mapped, base_mapped
+ arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped));
arena_stats_accum_zu(&astats->retained,
eset_npages_get(&arena->eset_retained) << LG_PAGE);
ecache_npages_get(&arena->ecache_retained) << LG_PAGE);
atomic_store_zu(&astats->edata_avail,
atomic_load_zu(&arena->edata_cache.count, ATOMIC_RELAXED),
@@ -131,8 +131,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_stats_accum_zu(&astats->metadata_thp, metadata_thp);
arena_stats_accum_zu(&astats->resident, base_resident +
(((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) +
eset_npages_get(&arena->eset_dirty) +
eset_npages_get(&arena->eset_muzzy)) << LG_PAGE)));
ecache_npages_get(&arena->ecache_dirty) +
ecache_npages_get(&arena->ecache_muzzy)) << LG_PAGE)));
arena_stats_accum_zu(&astats->abandoned_vm, atomic_load_zu(
&arena->stats.abandoned_vm, ATOMIC_RELAXED));
@@ -174,12 +174,12 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
for (pszind_t i = 0; i < SC_NPSIZES; i++) {
size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
retained_bytes;
dirty = eset_nextents_get(&arena->eset_dirty, i);
muzzy = eset_nextents_get(&arena->eset_muzzy, i);
retained = eset_nextents_get(&arena->eset_retained, i);
dirty_bytes = eset_nbytes_get(&arena->eset_dirty, i);
muzzy_bytes = eset_nbytes_get(&arena->eset_muzzy, i);
retained_bytes = eset_nbytes_get(&arena->eset_retained, i);
dirty = ecache_nextents_get(&arena->ecache_dirty, i);
muzzy = ecache_nextents_get(&arena->ecache_muzzy, i);
retained = ecache_nextents_get(&arena->ecache_retained, i);
dirty_bytes = ecache_nbytes_get(&arena->ecache_dirty, i);
muzzy_bytes = ecache_nbytes_get(&arena->ecache_muzzy, i);
retained_bytes = ecache_nbytes_get(&arena->ecache_retained, i);
atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED);
atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED);
@@ -226,11 +226,11 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
READ_ARENA_MUTEX_PROF_DATA(edata_cache.mtx,
arena_prof_mutex_extent_avail)
READ_ARENA_MUTEX_PROF_DATA(eset_dirty.mtx,
READ_ARENA_MUTEX_PROF_DATA(ecache_dirty.mtx,
arena_prof_mutex_extents_dirty)
READ_ARENA_MUTEX_PROF_DATA(eset_muzzy.mtx,
READ_ARENA_MUTEX_PROF_DATA(ecache_muzzy.mtx,
arena_prof_mutex_extents_muzzy)
READ_ARENA_MUTEX_PROF_DATA(eset_retained.mtx,
READ_ARENA_MUTEX_PROF_DATA(ecache_retained.mtx,
arena_prof_mutex_extents_retained)
READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx,
arena_prof_mutex_decay_dirty)
@@ -258,7 +258,7 @@ arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extents_dalloc(tsdn, arena, ehooks, &arena->eset_dirty, edata);
extents_dalloc(tsdn, arena, ehooks, &arena->ecache_dirty, edata);
if (arena_dirty_decay_ms_get(arena) == 0) {
arena_decay_dirty(tsdn, arena, false, true);
} else {
@@ -434,10 +434,11 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
szind_t szind = sz_size2index(usize);
size_t mapped_add;
bool commit = true;
edata_t *edata = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit);
edata_t *edata = extents_alloc(tsdn, arena, ehooks,
&arena->ecache_dirty, NULL, usize, sz_large_pad, alignment, false,
szind, zero, &commit);
if (edata == NULL && arena_may_have_muzzy(arena)) {
edata = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
edata = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy,
NULL, usize, sz_large_pad, alignment, false, szind, zero,
&commit);
}
@@ -606,10 +607,10 @@ arena_decay_backlog_update(arena_decay_t *decay, uint64_t nadvance_u64,
static void
arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
eset_t *eset, size_t current_npages, size_t npages_limit,
ecache_t *ecache, size_t current_npages, size_t npages_limit,
bool is_background_thread) {
if (current_npages > npages_limit) {
arena_decay_to_limit(tsdn, arena, decay, eset, false,
arena_decay_to_limit(tsdn, arena, decay, ecache, false,
npages_limit, current_npages - npages_limit,
is_background_thread);
}
@@ -641,8 +642,8 @@ arena_decay_epoch_advance_helper(arena_decay_t *decay, const nstime_t *time,
static void
arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
eset_t *eset, const nstime_t *time, bool is_background_thread) {
size_t current_npages = eset_npages_get(eset);
ecache_t *ecache, const nstime_t *time, bool is_background_thread) {
size_t current_npages = ecache_npages_get(ecache);
arena_decay_epoch_advance_helper(decay, time, current_npages);
size_t npages_limit = arena_decay_backlog_npages_limit(decay);
@@ -651,7 +652,7 @@ arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
current_npages;
if (!background_thread_enabled() || is_background_thread) {
arena_decay_try_purge(tsdn, arena, decay, eset,
arena_decay_try_purge(tsdn, arena, decay, ecache,
current_npages, npages_limit, is_background_thread);
}
}
@@ -708,15 +709,15 @@ arena_decay_ms_valid(ssize_t decay_ms) {
static bool
arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
eset_t *eset, bool is_background_thread) {
ecache_t *ecache, bool is_background_thread) {
malloc_mutex_assert_owner(tsdn, &decay->mtx);
/* Purge all or nothing if the option is disabled. */
ssize_t decay_ms = arena_decay_ms_read(decay);
if (decay_ms <= 0) {
if (decay_ms == 0) {
arena_decay_to_limit(tsdn, arena, decay, eset, false,
0, eset_npages_get(eset),
arena_decay_to_limit(tsdn, arena, decay, ecache, false,
0, ecache_npages_get(ecache),
is_background_thread);
}
return false;
@@ -751,11 +752,11 @@ arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
*/
bool advance_epoch = arena_decay_deadline_reached(decay, &time);
if (advance_epoch) {
arena_decay_epoch_advance(tsdn, arena, decay, eset, &time,
arena_decay_epoch_advance(tsdn, arena, decay, ecache, &time,
is_background_thread);
} else if (is_background_thread) {
arena_decay_try_purge(tsdn, arena, decay, eset,
eset_npages_get(eset),
arena_decay_try_purge(tsdn, arena, decay, ecache,
ecache_npages_get(ecache),
arena_decay_backlog_npages_limit(decay),
is_background_thread);
}
@@ -780,7 +781,7 @@ arena_muzzy_decay_ms_get(arena_t *arena) {
static bool
arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
eset_t *eset, ssize_t decay_ms) {
ecache_t *ecache, ssize_t decay_ms) {
if (!arena_decay_ms_valid(decay_ms)) {
return true;
}
@@ -795,7 +796,7 @@ arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
* arbitrary change during initial arena configuration.
*/
arena_decay_reinit(decay, decay_ms);
arena_maybe_decay(tsdn, arena, decay, eset, false);
arena_maybe_decay(tsdn, arena, decay, ecache, false);
malloc_mutex_unlock(tsdn, &decay->mtx);
return false;
@@ -805,19 +806,19 @@ bool
arena_dirty_decay_ms_set(tsdn_t *tsdn, arena_t *arena,
ssize_t decay_ms) {
return arena_decay_ms_set(tsdn, arena, &arena->decay_dirty,
&arena->eset_dirty, decay_ms);
&arena->ecache_dirty, decay_ms);
}
bool
arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena,
ssize_t decay_ms) {
return arena_decay_ms_set(tsdn, arena, &arena->decay_muzzy,
&arena->eset_muzzy, decay_ms);
&arena->ecache_muzzy, decay_ms);
}
static size_t
arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
ehooks_t *ehooks, eset_t *eset, size_t npages_limit,
ehooks_t *ehooks, ecache_t *ecache, size_t npages_limit,
size_t npages_decay_max, edata_list_t *decay_extents) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
@@ -826,7 +827,7 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
size_t nstashed = 0;
edata_t *edata;
while (nstashed < npages_decay_max &&
(edata = extents_evict(tsdn, arena, ehooks, eset, npages_limit))
(edata = extents_evict(tsdn, arena, ehooks, ecache, npages_limit))
!= NULL) {
edata_list_append(decay_extents, edata);
nstashed += edata_size_get(edata) >> LG_PAGE;
@@ -836,8 +837,8 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
static size_t
arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
arena_decay_t *decay, eset_t *eset, bool all, edata_list_t *decay_extents,
bool is_background_thread) {
arena_decay_t *decay, ecache_t *ecache, bool all,
edata_list_t *decay_extents, bool is_background_thread) {
size_t nmadvise, nunmapped;
size_t npurged;
@@ -856,7 +857,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
size_t npages = edata_size_get(edata) >> LG_PAGE;
npurged += npages;
edata_list_remove(decay_extents, edata);
switch (eset_state_get(eset)) {
switch (eset_state_get(&ecache->eset)) {
case extent_state_active:
not_reached();
case extent_state_dirty:
@@ -864,7 +865,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
!extent_purge_lazy_wrapper(tsdn, arena,
ehooks, edata, 0, edata_size_get(edata))) {
extents_dalloc(tsdn, arena, ehooks,
&arena->eset_muzzy, edata);
&arena->ecache_muzzy, edata);
arena_background_thread_inactivity_check(tsdn,
arena, is_background_thread);
break;
@@ -900,14 +901,14 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
/*
* npages_limit: Decay at most npages_decay_max pages without violating the
* invariant: (eset_npages_get(extents) >= npages_limit). We need an upper
* invariant: (ecache_npages_get(ecache) >= npages_limit). We need an upper
* bound on number of pages in order to prevent unbounded growth (namely in
* stashed), otherwise unbounded new pages could be added to extents during the
* current decay run, so that the purging thread never finishes.
*/
static void
arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
eset_t *eset, bool all, size_t npages_limit, size_t npages_decay_max,
ecache_t *ecache, bool all, size_t npages_limit, size_t npages_decay_max,
bool is_background_thread) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 1);
@@ -924,11 +925,11 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
edata_list_t decay_extents;
edata_list_init(&decay_extents);
size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, eset,
size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, ecache,
npages_limit, npages_decay_max, &decay_extents);
if (npurge != 0) {
size_t npurged = arena_decay_stashed(tsdn, arena, ehooks, decay,
eset, all, &decay_extents, is_background_thread);
ecache, all, &decay_extents, is_background_thread);
assert(npurged == npurge);
}
@@ -938,11 +939,11 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
static bool
arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
eset_t *eset, bool is_background_thread, bool all) {
ecache_t *ecache, bool is_background_thread, bool all) {
if (all) {
malloc_mutex_lock(tsdn, &decay->mtx);
arena_decay_to_limit(tsdn, arena, decay, eset, all, 0,
eset_npages_get(eset), is_background_thread);
arena_decay_to_limit(tsdn, arena, decay, ecache, all, 0,
ecache_npages_get(ecache), is_background_thread);
malloc_mutex_unlock(tsdn, &decay->mtx);
return false;
@@ -953,7 +954,7 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
return true;
}
bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, eset,
bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, ecache,
is_background_thread);
size_t npages_new;
if (epoch_advanced) {
@@ -975,18 +976,18 @@ static bool
arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
bool all) {
return arena_decay_impl(tsdn, arena, &arena->decay_dirty,
&arena->eset_dirty, is_background_thread, all);
&arena->ecache_dirty, is_background_thread, all);
}
static bool
arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
bool all) {
if (eset_npages_get(&arena->eset_muzzy) == 0 &&
if (ecache_npages_get(&arena->ecache_muzzy) == 0 &&
arena_muzzy_decay_ms_get(arena) <= 0) {
return false;
}
return arena_decay_impl(tsdn, arena, &arena->decay_muzzy,
&arena->eset_muzzy, is_background_thread, all);
&arena->ecache_muzzy, is_background_thread, all);
}
void
@@ -1157,7 +1158,7 @@ arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) {
ehooks_t *ehooks = arena_get_ehooks(arena);
edata_t *edata;
while ((edata = extents_evict(tsdn, arena, ehooks,
&arena->eset_retained, 0)) != NULL) {
&arena->ecache_retained, 0)) != NULL) {
extent_destroy_wrapper(tsdn, arena, ehooks, edata);
}
}
@@ -1173,8 +1174,8 @@ arena_destroy(tsd_t *tsd, arena_t *arena) {
* Furthermore, the caller (arena_i_destroy_ctl()) purged all cached
* extents, so only retained extents may remain.
*/
assert(eset_npages_get(&arena->eset_dirty) == 0);
assert(eset_npages_get(&arena->eset_muzzy) == 0);
assert(ecache_npages_get(&arena->ecache_dirty) == 0);
assert(ecache_npages_get(&arena->ecache_muzzy) == 0);
/* Deallocate retained memory. */
arena_destroy_retained(tsd_tsdn(tsd), arena);
@@ -1230,10 +1231,10 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard
szind_t szind = sz_size2index(bin_info->reg_size);
bool zero = false;
bool commit = true;
edata_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
edata_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->ecache_dirty,
NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit);
if (slab == NULL && arena_may_have_muzzy(arena)) {
slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
slab = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy,
NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero,
&commit);
}
@@ -1917,14 +1918,14 @@ arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit,
}
}
malloc_mutex_lock(tsd_tsdn(tsd), &arena->extent_grow_mtx);
malloc_mutex_lock(tsd_tsdn(tsd), &arena->ecache_grow.mtx);
if (old_limit != NULL) {
*old_limit = sz_pind2sz(arena->retain_grow_limit);
*old_limit = sz_pind2sz(arena->ecache_grow.limit);
}
if (new_limit != NULL) {
arena->retain_grow_limit = new_ind;
arena->ecache_grow.limit = new_ind;
}
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->extent_grow_mtx);
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->ecache_grow.mtx);
return false;
}
@@ -2016,14 +2017,14 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
* are likely to be reused soon after deallocation, and the cost of
* merging/splitting extents is non-trivial.
*/
if (eset_init(tsdn, &arena->eset_dirty, extent_state_dirty, true)) {
if (ecache_init(tsdn, &arena->ecache_dirty, extent_state_dirty, true)) {
goto label_error;
}
/*
* Coalesce muzzy extents immediately, because operations on them are in
* the critical path much less often than for dirty extents.
*/
if (eset_init(tsdn, &arena->eset_muzzy, extent_state_muzzy, false)) {
if (ecache_init(tsdn, &arena->ecache_muzzy, extent_state_muzzy, false)) {
goto label_error;
}
/*
@@ -2032,7 +2033,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
* coalescing), but also because operations on retained extents are not
* in the critical path.
*/
if (eset_init(tsdn, &arena->eset_retained, extent_state_retained,
if (ecache_init(tsdn, &arena->ecache_retained, extent_state_retained,
false)) {
goto label_error;
}
@@ -2046,10 +2047,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
goto label_error;
}
arena->extent_grow_next = sz_psz2ind(HUGEPAGE);
arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS);
if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow",
WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
if (ecache_grow_init(tsdn, &arena->ecache_grow)) {
goto label_error;
}
@@ -2187,14 +2185,14 @@ arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
void
arena_prefork2(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_prefork(tsdn, &arena->extent_grow_mtx);
ecache_grow_prefork(tsdn, &arena->ecache_grow);
}
void
arena_prefork3(tsdn_t *tsdn, arena_t *arena) {
eset_prefork(tsdn, &arena->eset_dirty);
eset_prefork(tsdn, &arena->eset_muzzy);
eset_prefork(tsdn, &arena->eset_retained);
ecache_prefork(tsdn, &arena->ecache_dirty);
ecache_prefork(tsdn, &arena->ecache_muzzy);
ecache_prefork(tsdn, &arena->ecache_retained);
}
void
@@ -2234,10 +2232,10 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
base_postfork_parent(tsdn, arena->base);
edata_cache_postfork_parent(tsdn, &arena->edata_cache);
eset_postfork_parent(tsdn, &arena->eset_dirty);
eset_postfork_parent(tsdn, &arena->eset_muzzy);
eset_postfork_parent(tsdn, &arena->eset_retained);
malloc_mutex_postfork_parent(tsdn, &arena->extent_grow_mtx);
ecache_postfork_parent(tsdn, &arena->ecache_dirty);
ecache_postfork_parent(tsdn, &arena->ecache_muzzy);
ecache_postfork_parent(tsdn, &arena->ecache_retained);
ecache_grow_postfork_parent(tsdn, &arena->ecache_grow);
malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx);
if (config_stats) {
@@ -2280,10 +2278,10 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
base_postfork_child(tsdn, arena->base);
edata_cache_postfork_child(tsdn, &arena->edata_cache);
eset_postfork_child(tsdn, &arena->eset_dirty);
eset_postfork_child(tsdn, &arena->eset_muzzy);
eset_postfork_child(tsdn, &arena->eset_retained);
malloc_mutex_postfork_child(tsdn, &arena->extent_grow_mtx);
ecache_postfork_child(tsdn, &arena->ecache_dirty);
ecache_postfork_child(tsdn, &arena->ecache_muzzy);
ecache_postfork_child(tsdn, &arena->ecache_retained);
ecache_grow_postfork_child(tsdn, &arena->ecache_grow);
malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx);
if (config_stats) {

12
src/background_thread.c
View File

@@ -114,7 +114,7 @@ decay_npurge_after_interval(arena_decay_t *decay, size_t interval) {
static uint64_t
arena_decay_compute_purge_interval_impl(tsdn_t *tsdn, arena_decay_t *decay,
eset_t *eset) {
ecache_t *ecache) {
if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
/* Use minimal interval if decay is contended. */
return BACKGROUND_THREAD_MIN_INTERVAL_NS;
@@ -130,7 +130,7 @@ arena_decay_compute_purge_interval_impl(tsdn_t *tsdn, arena_decay_t *decay,
uint64_t decay_interval_ns = nstime_ns(&decay->interval);
assert(decay_interval_ns > 0);
size_t npages = eset_npages_get(eset);
size_t npages = ecache_npages_get(ecache);
if (npages == 0) {
unsigned i;
for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) {
@@ -202,12 +202,12 @@ static uint64_t
arena_decay_compute_purge_interval(tsdn_t *tsdn, arena_t *arena) {
uint64_t i1, i2;
i1 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_dirty,
&arena->eset_dirty);
&arena->ecache_dirty);
if (i1 == BACKGROUND_THREAD_MIN_INTERVAL_NS) {
return i1;
}
i2 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_muzzy,
&arena->eset_muzzy);
&arena->ecache_muzzy);
return i1 < i2 ? i1 : i2;
}
@@ -717,8 +717,8 @@ background_thread_interval_check(tsdn_t *tsdn, arena_t *arena,
if (info->npages_to_purge_new > BACKGROUND_THREAD_NPAGES_THRESHOLD) {
should_signal = true;
} else if (unlikely(background_thread_indefinite_sleep(info)) &&
(eset_npages_get(&arena->eset_dirty) > 0 ||
eset_npages_get(&arena->eset_muzzy) > 0 ||
(ecache_npages_get(&arena->ecache_dirty) > 0 ||
ecache_npages_get(&arena->ecache_muzzy) > 0 ||
info->npages_to_purge_new > 0)) {
should_signal = true;
} else {

6
src/ctl.c
View File

@@ -3011,9 +3011,9 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
}
MUTEX_PROF_RESET(arena->large_mtx);
MUTEX_PROF_RESET(arena->edata_cache.mtx);
MUTEX_PROF_RESET(arena->eset_dirty.mtx);
MUTEX_PROF_RESET(arena->eset_muzzy.mtx);
MUTEX_PROF_RESET(arena->eset_retained.mtx);
MUTEX_PROF_RESET(arena->ecache_dirty.mtx);
MUTEX_PROF_RESET(arena->ecache_muzzy.mtx);
MUTEX_PROF_RESET(arena->ecache_retained.mtx);
MUTEX_PROF_RESET(arena->decay_dirty.mtx);
MUTEX_PROF_RESET(arena->decay_muzzy.mtx);
MUTEX_PROF_RESET(arena->tcache_ql_mtx);

54
src/ecache.c Normal file
View File

@@ -0,0 +1,54 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
bool
ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state,
bool delay_coalesce) {
if (malloc_mutex_init(&ecache->mtx, "extents", WITNESS_RANK_EXTENTS,
malloc_mutex_rank_exclusive)) {
return true;
}
eset_init(&ecache->eset, state, delay_coalesce);
return false;
}
void
ecache_prefork(tsdn_t *tsdn, ecache_t *ecache) {
malloc_mutex_prefork(tsdn, &ecache->mtx);
}
void
ecache_postfork_parent(tsdn_t *tsdn, ecache_t *ecache) {
malloc_mutex_postfork_parent(tsdn, &ecache->mtx);
}
void
ecache_postfork_child(tsdn_t *tsdn, ecache_t *ecache) {
malloc_mutex_postfork_child(tsdn, &ecache->mtx);
}
bool
ecache_grow_init(tsdn_t *tsdn, ecache_grow_t *ecache_grow) {
ecache_grow->next = sz_psz2ind(HUGEPAGE);
ecache_grow->limit = sz_psz2ind(SC_LARGE_MAXCLASS);
if (malloc_mutex_init(&ecache_grow->mtx, "extent_grow",
WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
return true;
}
return false;
}
void
ecache_grow_prefork(tsdn_t *tsdn, ecache_grow_t *ecache_grow) {
malloc_mutex_prefork(tsdn, &ecache_grow->mtx);
}
void
ecache_grow_postfork_parent(tsdn_t *tsdn, ecache_grow_t *ecache_grow) {
malloc_mutex_postfork_parent(tsdn, &ecache_grow->mtx);
}
void
ecache_grow_postfork_child(tsdn_t *tsdn, ecache_grow_t *ecache_grow) {
malloc_mutex_postfork_child(tsdn, &ecache_grow->mtx);
}

44
src/eset.c
View File

@@ -8,13 +8,9 @@
const bitmap_info_t eset_bitmap_info =
BITMAP_INFO_INITIALIZER(SC_NPSIZES+1);
bool
eset_init(tsdn_t *tsdn, eset_t *eset, extent_state_t state,
void
eset_init(eset_t *eset, extent_state_t state,
bool delay_coalesce) {
if (malloc_mutex_init(&eset->mtx, "extents", WITNESS_RANK_EXTENTS,
malloc_mutex_rank_exclusive)) {
return true;
}
for (unsigned i = 0; i < SC_NPSIZES + 1; i++) {
edata_heap_new(&eset->heaps[i]);
}
@@ -23,7 +19,6 @@ eset_init(tsdn_t *tsdn, eset_t *eset, extent_state_t state,
atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED);
eset->state = state;
eset->delay_coalesce = delay_coalesce;
return false;
}
extent_state_t
@@ -63,8 +58,7 @@ eset_stats_sub(eset_t *eset, pszind_t pind, size_t sz) {
}
void
eset_insert_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) {
malloc_mutex_assert_owner(tsdn, &eset->mtx);
eset_insert(eset_t *eset, edata_t *edata) {
assert(edata_state_get(edata) == eset->state);
size_t size = edata_size_get(edata);
@@ -94,8 +88,7 @@ eset_insert_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) {
}
void
eset_remove_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) {
malloc_mutex_assert_owner(tsdn, &eset->mtx);
eset_remove(eset_t *eset, edata_t *edata) {
assert(edata_state_get(edata) == eset->state);
size_t size = edata_size_get(edata);
@@ -114,9 +107,13 @@ eset_remove_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) {
edata_list_remove(&eset->lru, edata);
size_t npages = size >> LG_PAGE;
/*
* As in eset_insert_locked, we hold eset->mtx and so don't need atomic
* As in eset_insert, we hold eset->mtx and so don't need atomic
* operations for updating eset->npages.
*/
/*
* This class is not thread-safe in general; we rely on external
* synchronization for all mutating operations.
*/
size_t cur_extents_npages =
atomic_load_zu(&eset->npages, ATOMIC_RELAXED);
assert(cur_extents_npages >= npages);
@@ -166,7 +163,7 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size,
* large enough.
*/
static edata_t *
eset_first_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t size) {
eset_first_fit(eset_t *eset, size_t size) {
edata_t *ret = NULL;
pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(size));
@@ -211,16 +208,14 @@ eset_first_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t size) {
}
edata_t *
eset_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t esize, size_t alignment) {
malloc_mutex_assert_owner(tsdn, &eset->mtx);
eset_fit(eset_t *eset, size_t esize, size_t alignment) {
size_t max_size = esize + PAGE_CEILING(alignment) - PAGE;
/* Beware size_t wrap-around. */
if (max_size < esize) {
return NULL;
}
edata_t *edata = eset_first_fit_locked(tsdn, eset, max_size);
edata_t *edata = eset_first_fit(eset, max_size);
if (alignment > PAGE && edata == NULL) {
/*
@@ -233,18 +228,3 @@ eset_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t esize, size_t alignment) {
return edata;
}
void
eset_prefork(tsdn_t *tsdn, eset_t *eset) {
malloc_mutex_prefork(tsdn, &eset->mtx);
}
void
eset_postfork_parent(tsdn_t *tsdn, eset_t *eset) {
malloc_mutex_postfork_parent(tsdn, &eset->mtx);
}
void
eset_postfork_child(tsdn_t *tsdn, eset_t *eset) {
malloc_mutex_postfork_child(tsdn, &eset->mtx);
}

226
src/extent2.c
View File

@@ -45,13 +45,13 @@ static atomic_zu_t highpages;
static void extent_deregister(tsdn_t *tsdn, edata_t *edata);
static edata_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, void *new_addr, size_t usize, size_t pad, size_t alignment,
ecache_t *ecache, void *new_addr, size_t usize, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained);
static edata_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata,
ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata,
bool *coalesced, bool growing_retained);
static void extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, edata_t *edata, bool growing_retained);
ecache_t *ecache, edata_t *edata, bool growing_retained);
/******************************************************************************/
@@ -165,22 +165,22 @@ extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
static bool
extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata) {
rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata) {
edata_state_set(edata, extent_state_active);
bool coalesced;
edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, eset,
edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, ecache,
edata, &coalesced, false);
edata_state_set(edata, eset_state_get(eset));
edata_state_set(edata, eset_state_get(&ecache->eset));
if (!coalesced) {
return true;
}
eset_insert_locked(tsdn, eset, edata);
eset_insert(&ecache->eset, edata);
return false;
}
edata_t *
extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit) {
assert(size + pad != 0);
@@ -188,14 +188,14 @@ extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
edata_t *edata = extent_recycle(tsdn, arena, ehooks, eset, new_addr,
edata_t *edata = extent_recycle(tsdn, arena, ehooks, ecache, new_addr,
size, pad, alignment, slab, szind, zero, commit, false);
assert(edata == NULL || edata_dumpable_get(edata));
return edata;
}
void
extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
edata_t *edata) {
assert(edata_base_get(edata) != NULL);
assert(edata_size_get(edata) != 0);
@@ -206,16 +206,16 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
edata_addr_set(edata, edata_base_get(edata));
edata_zeroed_set(edata, false);
extent_record(tsdn, arena, ehooks, eset, edata, false);
extent_record(tsdn, arena, ehooks, ecache, edata, false);
}
edata_t *
extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
size_t npages_min) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
malloc_mutex_lock(tsdn, &eset->mtx);
malloc_mutex_lock(tsdn, &ecache->mtx);
/*
* Get the LRU coalesced extent, if any. If coalescing was delayed,
@@ -224,24 +224,23 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
edata_t *edata;
while (true) {
/* Get the LRU extent, if any. */
edata = edata_list_first(&eset->lru);
edata = edata_list_first(&ecache->eset.lru);
if (edata == NULL) {
goto label_return;
}
/* Check the eviction limit. */
size_t extents_npages = atomic_load_zu(&eset->npages,
ATOMIC_RELAXED);
size_t extents_npages = ecache_npages_get(ecache);
if (extents_npages <= npages_min) {
edata = NULL;
goto label_return;
}
eset_remove_locked(tsdn, eset, edata);
if (!eset->delay_coalesce) {
eset_remove(&ecache->eset, edata);
if (!ecache->eset.delay_coalesce) {
break;
}
/* Try to coalesce. */
if (extent_try_delayed_coalesce(tsdn, arena, ehooks, rtree_ctx,
eset, edata)) {
ecache, edata)) {
break;
}
/*
@@ -254,7 +253,7 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
* Either mark the extent active or deregister it to protect against
* concurrent operations.
*/
switch (eset_state_get(eset)) {
switch (eset_state_get(&ecache->eset)) {
case extent_state_active:
not_reached();
case extent_state_dirty:
@@ -269,7 +268,7 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
}
label_return:
malloc_mutex_unlock(tsdn, &eset->mtx);
malloc_mutex_unlock(tsdn, &ecache->mtx);
return edata;
}
@@ -278,8 +277,8 @@ label_return:
* indicates OOM), e.g. when trying to split an existing extent.
*/
static void
extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
edata_t *edata, bool growing_retained) {
extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
ecache_t *ecache, edata_t *edata, bool growing_retained) {
size_t sz = edata_size_get(edata);
if (config_stats) {
arena_stats_accum_zu(&arena->stats.abandoned_vm, sz);
@@ -288,7 +287,7 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
* Leak extent after making sure its pages have already been purged, so
* that this is only a virtual memory leak.
*/
if (eset_state_get(eset) == extent_state_dirty) {
if (eset_state_get(&ecache->eset) == extent_state_dirty) {
if (extent_purge_lazy_impl(tsdn, arena, ehooks, edata, 0, sz,
growing_retained)) {
extent_purge_forced_impl(tsdn, arena, ehooks, edata, 0,
@@ -299,30 +298,30 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
}
static void
extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache,
edata_t *edata) {
assert(edata_arena_ind_get(edata) == arena_ind_get(arena));
assert(edata_state_get(edata) == extent_state_active);
edata_state_set(edata, eset_state_get(eset));
eset_insert_locked(tsdn, eset, edata);
edata_state_set(edata, eset_state_get(&ecache->eset));
eset_insert(&ecache->eset, edata);
}
static void
extent_deactivate(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
extent_deactivate(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache,
edata_t *edata) {
malloc_mutex_lock(tsdn, &eset->mtx);
extent_deactivate_locked(tsdn, arena, eset, edata);
malloc_mutex_unlock(tsdn, &eset->mtx);
malloc_mutex_lock(tsdn, &ecache->mtx);
extent_deactivate_locked(tsdn, arena, ecache, edata);
malloc_mutex_unlock(tsdn, &ecache->mtx);
}
static void
extent_activate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
extent_activate_locked(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache,
edata_t *edata) {
assert(edata_arena_ind_get(edata) == arena_ind_get(arena));
assert(edata_state_get(edata) == eset_state_get(eset));
assert(edata_state_get(edata) == eset_state_get(&ecache->eset));
eset_remove_locked(tsdn, eset, edata);
eset_remove(&ecache->eset, edata);
edata_state_set(edata, extent_state_active);
}
@@ -515,12 +514,12 @@ extent_deregister_no_gdump_sub(tsdn_t *tsdn, edata_t *edata) {
}
/*
* Tries to find and remove an extent from eset that can be used for the
* Tries to find and remove an extent from ecache that can be used for the
* given allocation request.
*/
static edata_t *
extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
rtree_ctx_t *rtree_ctx, ecache_t *ecache, void *new_addr, size_t size,
size_t pad, size_t alignment, bool slab, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
@@ -543,7 +542,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
}
size_t esize = size + pad;
malloc_mutex_lock(tsdn, &eset->mtx);
malloc_mutex_lock(tsdn, &ecache->mtx);
edata_t *edata;
if (new_addr != NULL) {
edata = extent_lock_edata_from_addr(tsdn, rtree_ctx, new_addr,
@@ -557,21 +556,22 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
assert(edata_base_get(edata) == new_addr);
if (edata_arena_ind_get(edata) != arena_ind_get(arena)
|| edata_size_get(edata) < esize
|| edata_state_get(edata) != eset_state_get(eset)) {
|| edata_state_get(edata)
!= eset_state_get(&ecache->eset)) {
edata = NULL;
}
extent_unlock_edata(tsdn, unlock_edata);
}
} else {
edata = eset_fit_locked(tsdn, eset, esize, alignment);
edata = eset_fit(&ecache->eset, esize, alignment);
}
if (edata == NULL) {
malloc_mutex_unlock(tsdn, &eset->mtx);
malloc_mutex_unlock(tsdn, &ecache->mtx);
return NULL;
}
extent_activate_locked(tsdn, arena, eset, edata);
malloc_mutex_unlock(tsdn, &eset->mtx);
extent_activate_locked(tsdn, arena, ecache, edata);
malloc_mutex_unlock(tsdn, &ecache->mtx);
return edata;
}
@@ -580,7 +580,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
* Given an allocation request and an extent guaranteed to be able to satisfy
* it, this splits off lead and trail extents, leaving edata pointing to an
* extent satisfying the allocation.
* This function doesn't put lead or trail into any eset_t; it's the caller's
* This function doesn't put lead or trail into any ecache; it's the caller's
* job to ensure that they can be reused.
*/
typedef enum {
@@ -676,11 +676,11 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
* This fulfills the indicated allocation request out of the given extent (which
* the caller should have ensured was big enough). If there's any unused space
* before or after the resulting allocation, that space is given its own extent
* and put back into eset.
* and put back into ecache.
*/
static edata_t *
extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
rtree_ctx_t *rtree_ctx, ecache_t *ecache, void *new_addr, size_t size,
size_t pad, size_t alignment, bool slab, szind_t szind, edata_t *edata,
bool growing_retained) {
edata_t *lead;
@@ -697,19 +697,19 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
&& !opt_retain) {
/*
* Split isn't supported (implies Windows w/o retain). Avoid
* leaking the eset.
* leaking the extent.
*/
assert(to_leak != NULL && lead == NULL && trail == NULL);
extent_deactivate(tsdn, arena, eset, to_leak);
extent_deactivate(tsdn, arena, ecache, to_leak);
return NULL;
}
if (result == extent_split_interior_ok) {
if (lead != NULL) {
extent_deactivate(tsdn, arena, eset, lead);
extent_deactivate(tsdn, arena, ecache, lead);
}
if (trail != NULL) {
extent_deactivate(tsdn, arena, eset, trail);
extent_deactivate(tsdn, arena, ecache, trail);
}
return edata;
} else {
@@ -724,7 +724,7 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
if (to_leak != NULL) {
void *leak = edata_base_get(to_leak);
extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, ehooks, eset, to_leak,
extents_abandon_vm(tsdn, arena, ehooks, ecache, to_leak,
growing_retained);
assert(extent_lock_edata_from_addr(tsdn, rtree_ctx, leak,
false) == NULL);
@@ -736,10 +736,10 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
/*
* Tries to satisfy the given allocation request by reusing one of the extents
* in the given eset_t.
* in the given ecache_t.
*/
static edata_t *
extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
@@ -752,13 +752,13 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
edata_t *edata = extent_recycle_extract(tsdn, arena, ehooks,
rtree_ctx, eset, new_addr, size, pad, alignment, slab,
rtree_ctx, ecache, new_addr, size, pad, alignment, slab,
growing_retained);
if (edata == NULL) {
return NULL;
}
edata = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, eset,
edata = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, ecache,
new_addr, size, pad, alignment, slab, szind, edata,
growing_retained);
if (edata == NULL) {
@@ -768,7 +768,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
if (*commit && !edata_committed_get(edata)) {
if (extent_commit_impl(tsdn, arena, ehooks, edata, 0,
edata_size_get(edata), growing_retained)) {
extent_record(tsdn, arena, ehooks, eset, edata,
extent_record(tsdn, arena, ehooks, ecache, edata,
growing_retained);
return NULL;
}
@@ -810,7 +810,7 @@ static edata_t *
extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
bool *zero, bool *commit) {
malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx);
malloc_mutex_assert_owner(tsdn, &arena->ecache_grow.mtx);
assert(pad == 0 || !slab);
assert(!*zero || !slab);
@@ -825,15 +825,15 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
* satisfy this request.
*/
pszind_t egn_skip = 0;
size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
size_t alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip);
while (alloc_size < alloc_size_min) {
egn_skip++;
if (arena->extent_grow_next + egn_skip >=
if (arena->ecache_grow.next + egn_skip >=
sz_psz2ind(SC_LARGE_MAXCLASS)) {
/* Outside legal range. */
goto label_err;
}
alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip);
}
edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache,
@@ -881,11 +881,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
if (result == extent_split_interior_ok) {
if (lead != NULL) {
extent_record(tsdn, arena, ehooks,
&arena->eset_retained, lead, true);
&arena->ecache_retained, lead, true);
}
if (trail != NULL) {
extent_record(tsdn, arena, ehooks,
&arena->eset_retained, trail, true);
&arena->ecache_retained, trail, true);
}
} else {
/*
@@ -898,12 +898,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_gdump_add(tsdn, to_salvage);
}
extent_record(tsdn, arena, ehooks,
&arena->eset_retained, to_salvage, true);
&arena->ecache_retained, to_salvage, true);
}
if (to_leak != NULL) {
extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, ehooks,
&arena->eset_retained, to_leak, true);
&arena->ecache_retained, to_leak, true);
}
goto label_err;
}
@@ -912,7 +912,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
if (extent_commit_impl(tsdn, arena, ehooks, edata, 0,
edata_size_get(edata), true)) {
extent_record(tsdn, arena, ehooks,
&arena->eset_retained, edata, true);
&arena->ecache_retained, edata, true);
goto label_err;
}
/* A successful commit should return zeroed memory. */
@@ -930,14 +930,14 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
* Increment extent_grow_next if doing so wouldn't exceed the allowed
* range.
*/
if (arena->extent_grow_next + egn_skip + 1 <=
arena->retain_grow_limit) {
arena->extent_grow_next += egn_skip + 1;
if (arena->ecache_grow.next + egn_skip + 1 <=
arena->ecache_grow.limit) {
arena->ecache_grow.next += egn_skip + 1;
} else {
arena->extent_grow_next = arena->retain_grow_limit;
arena->ecache_grow.next = arena->ecache_grow.limit;
}
/* All opportunities for failure are past. */
malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx);
if (config_prof) {
/* Adjust gdump stats now that extent is final size. */
@@ -962,7 +962,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
return edata;
label_err:
malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx);
return NULL;
}
@@ -973,13 +973,13 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
assert(size != 0);
assert(alignment != 0);
malloc_mutex_lock(tsdn, &arena->extent_grow_mtx);
malloc_mutex_lock(tsdn, &arena->ecache_grow.mtx);
edata_t *edata = extent_recycle(tsdn, arena, ehooks,
&arena->eset_retained, new_addr, size, pad, alignment, slab,
&arena->ecache_retained, new_addr, size, pad, alignment, slab,
szind, zero, commit, true);
if (edata != NULL) {
malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx);
if (config_prof) {
extent_gdump_add(tsdn, edata);
}
@@ -988,9 +988,9 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
alignment, slab, szind, zero, commit);
/* extent_grow_retained() always releases extent_grow_mtx. */
} else {
malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx);
}
malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx);
malloc_mutex_assert_not_owner(tsdn, &arena->ecache_grow.mtx);
return edata;
}
@@ -1054,7 +1054,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
}
static bool
extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner,
extent_can_coalesce(arena_t *arena, ecache_t *ecache, const edata_t *inner,
const edata_t *outer) {
assert(edata_arena_ind_get(inner) == arena_ind_get(arena));
if (edata_arena_ind_get(outer) != arena_ind_get(arena)) {
@@ -1062,7 +1062,7 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner,
}
assert(edata_state_get(inner) == extent_state_active);
if (edata_state_get(outer) != eset->state) {
if (edata_state_get(outer) != ecache->eset.state) {
return false;
}
@@ -1074,19 +1074,20 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner,
}
static bool
extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
edata_t *inner, edata_t *outer, bool forward, bool growing_retained) {
assert(extent_can_coalesce(arena, eset, inner, outer));
extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
ecache_t *ecache, edata_t *inner, edata_t *outer, bool forward,
bool growing_retained) {
assert(extent_can_coalesce(arena, ecache, inner, outer));
extent_activate_locked(tsdn, arena, eset, outer);
extent_activate_locked(tsdn, arena, ecache, outer);
malloc_mutex_unlock(tsdn, &eset->mtx);
malloc_mutex_unlock(tsdn, &ecache->mtx);
bool err = extent_merge_impl(tsdn, arena, ehooks,
forward ? inner : outer, forward ? outer : inner, growing_retained);
malloc_mutex_lock(tsdn, &eset->mtx);
malloc_mutex_lock(tsdn, &ecache->mtx);
if (err) {
extent_deactivate_locked(tsdn, arena, eset, outer);
extent_deactivate_locked(tsdn, arena, ecache, outer);
}
return err;
@@ -1094,7 +1095,7 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
static edata_t *
extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced,
rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced,
bool growing_retained, bool inactive_only) {
/*
* We avoid checking / locking inactive neighbors for large size
@@ -1114,19 +1115,19 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
edata_past_get(edata), inactive_only);
if (next != NULL) {
/*
* eset->mtx only protects against races for
* like-state eset, so call extent_can_coalesce()
* ecache->mtx only protects against races for
* like-state extents, so call extent_can_coalesce()
* before releasing next's pool lock.
*/
bool can_coalesce = extent_can_coalesce(arena, eset,
bool can_coalesce = extent_can_coalesce(arena, ecache,
edata, next);
extent_unlock_edata(tsdn, next);
if (can_coalesce && !extent_coalesce(tsdn, arena,
ehooks, eset, edata, next, true,
ehooks, ecache, edata, next, true,
growing_retained)) {
if (eset->delay_coalesce) {
if (ecache->eset.delay_coalesce) {
/* Do minimal coalescing. */
*coalesced = true;
return edata;
@@ -1139,15 +1140,15 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
edata_t *prev = extent_lock_edata_from_addr(tsdn, rtree_ctx,
edata_before_get(edata), inactive_only);
if (prev != NULL) {
bool can_coalesce = extent_can_coalesce(arena, eset,
bool can_coalesce = extent_can_coalesce(arena, ecache,
edata, prev);
extent_unlock_edata(tsdn, prev);
if (can_coalesce && !extent_coalesce(tsdn, arena,
ehooks, eset, edata, prev, false,
ehooks, ecache, edata, prev, false,
growing_retained)) {
edata = prev;
if (eset->delay_coalesce) {
if (ecache->eset.delay_coalesce) {
/* Do minimal coalescing. */
*coalesced = true;
return edata;
@@ -1157,7 +1158,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
}
} while (again);
if (eset->delay_coalesce) {
if (ecache->eset.delay_coalesce) {
*coalesced = false;
}
return edata;
@@ -1165,35 +1166,35 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
static edata_t *
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced,
rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced,
bool growing_retained) {
return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, ecache,
edata, coalesced, growing_retained, false);
}
static edata_t *
extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced,
rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced,
bool growing_retained) {
return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, ecache,
edata, coalesced, growing_retained, true);
}
/*
* Does the metadata management portions of putting an unused extent into the
* given eset_t (coalesces, deregisters slab interiors, the heap operations).
* given ecache_t (coalesces, deregisters slab interiors, the heap operations).
*/
static void
extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
edata_t *edata, bool growing_retained) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
assert((eset_state_get(eset) != extent_state_dirty &&
eset_state_get(eset) != extent_state_muzzy) ||
assert((eset_state_get(&ecache->eset) != extent_state_dirty &&
eset_state_get(&ecache->eset) != extent_state_muzzy) ||
!edata_zeroed_get(edata));
malloc_mutex_lock(tsdn, &eset->mtx);
malloc_mutex_lock(tsdn, &ecache->mtx);
edata_szind_set(edata, SC_NSIZES);
if (edata_slab_get(edata)) {
@@ -1204,29 +1205,29 @@ extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
assert(rtree_edata_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), true) == edata);
if (!eset->delay_coalesce) {
if (!ecache->eset.delay_coalesce) {
edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx,
eset, edata, NULL, growing_retained);
ecache, edata, NULL, growing_retained);
} else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) {
assert(eset == &arena->eset_dirty);
/* Always coalesce large eset eagerly. */
assert(ecache == &arena->ecache_dirty);
/* Always coalesce large extents eagerly. */
bool coalesced;
do {
assert(edata_state_get(edata) == extent_state_active);
edata = extent_try_coalesce_large(tsdn, arena, ehooks,
rtree_ctx, eset, edata, &coalesced,
rtree_ctx, ecache, edata, &coalesced,
growing_retained);
} while (coalesced);
if (edata_size_get(edata) >= oversize_threshold) {
/* Shortcut to purge the oversize extent eagerly. */
malloc_mutex_unlock(tsdn, &eset->mtx);
malloc_mutex_unlock(tsdn, &ecache->mtx);
arena_decay_extent(tsdn, arena, ehooks, edata);
return;
}
}
extent_deactivate_locked(tsdn, arena, eset, edata);
extent_deactivate_locked(tsdn, arena, ecache, edata);
malloc_mutex_unlock(tsdn, &eset->mtx);
malloc_mutex_unlock(tsdn, &ecache->mtx);
}
void
@@ -1312,7 +1313,8 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_gdump_sub(tsdn, edata);
}
extent_record(tsdn, arena, ehooks, &arena->eset_retained, edata, false);
extent_record(tsdn, arena, ehooks, &arena->ecache_retained, edata,
false);
}
void

4
src/large.c
View File

@@ -149,10 +149,10 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
bool commit = true;
edata_t *trail;
bool new_mapping;
if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->ecache_dirty,
edata_past_get(edata), trailsize, 0, CACHELINE, false, SC_NSIZES,
&is_zeroed_trail, &commit)) != NULL
|| (trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
|| (trail = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy,
edata_past_get(edata), trailsize, 0, CACHELINE, false, SC_NSIZES,
&is_zeroed_trail, &commit)) != NULL) {
if (config_stats) {