From bb70df8e5babcf2779230d40b6a34fb04187c818 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Thu, 12 Dec 2019 16:25:24 -0800 Subject: [PATCH] Extent refactor: Introduce ecache module. This will eventually completely wrap the eset, and handle concurrency, allocation, and deallocation. For now, we only pull out the mutex from the eset. --- Makefile.in | 1 + include/jemalloc/internal/arena_structs.h | 28 +-- include/jemalloc/internal/ecache.h | 59 +++++ include/jemalloc/internal/eset.h | 47 +--- include/jemalloc/internal/extent2.h | 8 +- .../projects/vc2015/jemalloc/jemalloc.vcxproj | 1 + .../projects/vc2017/jemalloc/jemalloc.vcxproj | 1 + src/arena.c | 156 ++++++------ src/background_thread.c | 12 +- src/ctl.c | 6 +- src/ecache.c | 54 +++++ src/eset.c | 44 +--- src/extent2.c | 226 +++++++++--------- src/large.c | 4 +- test/unit/retained.c | 2 +- 15 files changed, 354 insertions(+), 295 deletions(-) create mode 100644 include/jemalloc/internal/ecache.h create mode 100644 src/ecache.c diff --git a/Makefile.in b/Makefile.in index f75ae4b6..71458487 100644 --- a/Makefile.in +++ b/Makefile.in @@ -104,6 +104,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/ckh.c \ $(srcroot)src/ctl.c \ $(srcroot)src/div.c \ + $(srcroot)src/ecache.c \ $(srcroot)src/edata.c \ $(srcroot)src/edata_cache.c \ $(srcroot)src/ehooks.c \ diff --git a/include/jemalloc/internal/arena_structs.h b/include/jemalloc/internal/arena_structs.h index 38c8b27c..48d13b8c 100644 --- a/include/jemalloc/internal/arena_structs.h +++ b/include/jemalloc/internal/arena_structs.h @@ -5,8 +5,8 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bin.h" #include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/ecache.h" #include "jemalloc/internal/edata_cache.h" -#include "jemalloc/internal/eset.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" @@ -53,7 +53,7 @@ struct arena_decay_s { /* * Number of unpurged pages at beginning of current epoch. During epoch * advancement we use the delta between arena->decay_*.nunpurged and - * eset_npages_get(&arena->extents_*) to determine how many dirty pages, + * ecache_npages_get(&arena->ecache_*) to determine how many dirty pages, * if any, were generated. */ size_t nunpurged; @@ -155,9 +155,9 @@ struct arena_s { * * Synchronization: internal. */ - eset_t eset_dirty; - eset_t eset_muzzy; - eset_t eset_retained; + ecache_t ecache_dirty; + ecache_t ecache_muzzy; + ecache_t ecache_retained; /* * Decay-based purging state, responsible for scheduling extent state @@ -168,22 +168,8 @@ struct arena_s { arena_decay_t decay_dirty; /* dirty --> muzzy */ arena_decay_t decay_muzzy; /* muzzy --> retained */ - /* - * Next extent size class in a growing series to use when satisfying a - * request via the extent hooks (only if opt_retain). This limits the - * number of disjoint virtual memory ranges so that extent merging can - * be effective even if multiple arenas' extent allocation requests are - * highly interleaved. - * - * retain_grow_limit is the max allowed size ind to expand (unless the - * required size is greater). Default is no limit, and controlled - * through mallctl only. - * - * Synchronization: extent_grow_mtx - */ - pszind_t extent_grow_next; - pszind_t retain_grow_limit; - malloc_mutex_t extent_grow_mtx; + /* The grow info for the retained ecache. */ + ecache_grow_t ecache_grow; /* The source of edata_t objects. */ edata_cache_t edata_cache; diff --git a/include/jemalloc/internal/ecache.h b/include/jemalloc/internal/ecache.h new file mode 100644 index 00000000..70857203 --- /dev/null +++ b/include/jemalloc/internal/ecache.h @@ -0,0 +1,59 @@ +#ifndef JEMALLOC_INTERNAL_ECACHE_H +#define JEMALLOC_INTERNAL_ECACHE_H + +#include "jemalloc/internal/eset.h" +#include "jemalloc/internal/mutex.h" + +typedef struct ecache_s ecache_t; +struct ecache_s { + malloc_mutex_t mtx; + eset_t eset; +}; + +typedef struct ecache_grow_s ecache_grow_t; +struct ecache_grow_s { + /* + * Next extent size class in a growing series to use when satisfying a + * request via the extent hooks (only if opt_retain). This limits the + * number of disjoint virtual memory ranges so that extent merging can + * be effective even if multiple arenas' extent allocation requests are + * highly interleaved. + * + * retain_grow_limit is the max allowed size ind to expand (unless the + * required size is greater). Default is no limit, and controlled + * through mallctl only. + * + * Synchronization: extent_grow_mtx + */ + pszind_t next; + pszind_t limit; + malloc_mutex_t mtx; +}; + +static inline size_t +ecache_npages_get(ecache_t *ecache) { + return eset_npages_get(&ecache->eset); +} +/* Get the number of extents in the given page size index. */ +static inline size_t +ecache_nextents_get(ecache_t *ecache, pszind_t ind) { + return eset_nextents_get(&ecache->eset, ind); +} +/* Get the sum total bytes of the extents in the given page size index. */ +static inline size_t +ecache_nbytes_get(ecache_t *ecache, pszind_t ind) { + return eset_nbytes_get(&ecache->eset, ind); +} + +bool ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, + bool delay_coalesce); +void ecache_prefork(tsdn_t *tsdn, ecache_t *ecache); +void ecache_postfork_parent(tsdn_t *tsdn, ecache_t *ecache); +void ecache_postfork_child(tsdn_t *tsdn, ecache_t *ecache); + +bool ecache_grow_init(tsdn_t *tsdn, ecache_grow_t *ecache_grow); +void ecache_grow_prefork(tsdn_t *tsdn, ecache_grow_t *ecache_grow); +void ecache_grow_postfork_parent(tsdn_t *tsdn, ecache_grow_t *ecache_grow); +void ecache_grow_postfork_child(tsdn_t *tsdn, ecache_grow_t *ecache_grow); + +#endif /* JEMALLOC_INTERNAL_ECACHE_H */ diff --git a/include/jemalloc/internal/eset.h b/include/jemalloc/internal/eset.h index e76257af..bbc6b5cd 100644 --- a/include/jemalloc/internal/eset.h +++ b/include/jemalloc/internal/eset.h @@ -9,42 +9,25 @@ /* * An eset ("extent set") is a quantized collection of extents, with built-in * LRU queue. + * + * This class is not thread-safe; synchronization must be done externally if + * there are mutating operations. One exception is the stats counters, which + * may be read without any locking. */ typedef struct eset_s eset_t; struct eset_s { - malloc_mutex_t mtx; - - /* - * Quantized per size class heaps of extents. - * - * Synchronization: mtx. - */ + /* Quantized per size class heaps of extents. */ edata_heap_t heaps[SC_NPSIZES + 1]; atomic_zu_t nextents[SC_NPSIZES + 1]; atomic_zu_t nbytes[SC_NPSIZES + 1]; - /* - * Bitmap for which set bits correspond to non-empty heaps. - * - * Synchronization: mtx. - */ + /* Bitmap for which set bits correspond to non-empty heaps. */ bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)]; - /* - * LRU of all extents in heaps. - * - * Synchronization: mtx. - */ + /* LRU of all extents in heaps. */ edata_list_t lru; - /* - * Page sum for all extents in heaps. - * - * The synchronization here is a little tricky. Modifications to npages - * must hold mtx, but reads need not (though, a reader who sees npages - * without holding the mutex can't assume anything about the rest of the - * state of the eset_t). - */ + /* Page sum for all extents in heaps. */ atomic_zu_t npages; /* All stored extents must be in the same state. */ @@ -57,8 +40,7 @@ struct eset_s { bool delay_coalesce; }; -bool eset_init(tsdn_t *tsdn, eset_t *eset, extent_state_t state, - bool delay_coalesce); +void eset_init(eset_t *eset, extent_state_t state, bool delay_coalesce); extent_state_t eset_state_get(const eset_t *eset); size_t eset_npages_get(eset_t *eset); @@ -67,17 +49,12 @@ size_t eset_nextents_get(eset_t *eset, pszind_t ind); /* Get the sum total bytes of the extents in the given page size index. */ size_t eset_nbytes_get(eset_t *eset, pszind_t ind); -void eset_insert_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata); -void eset_remove_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata); +void eset_insert(eset_t *eset, edata_t *edata); +void eset_remove(eset_t *eset, edata_t *edata); /* * Select an extent from this eset of the given size and alignment. Returns * null if no such item could be found. */ -edata_t *eset_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t esize, - size_t alignment); - -void eset_prefork(tsdn_t *tsdn, eset_t *eset); -void eset_postfork_parent(tsdn_t *tsdn, eset_t *eset); -void eset_postfork_child(tsdn_t *tsdn, eset_t *eset); +edata_t *eset_fit(eset_t *eset, size_t esize, size_t alignment); #endif /* JEMALLOC_INTERNAL_ESET_H */ diff --git a/include/jemalloc/internal/extent2.h b/include/jemalloc/internal/extent2.h index 08443366..80e789e4 100644 --- a/include/jemalloc/internal/extent2.h +++ b/include/jemalloc/internal/extent2.h @@ -1,8 +1,8 @@ #ifndef JEMALLOC_INTERNAL_EXTENT2_H #define JEMALLOC_INTERNAL_EXTENT2_H +#include "jemalloc/internal/ecache.h" #include "jemalloc/internal/ehooks.h" -#include "jemalloc/internal/eset.h" #include "jemalloc/internal/ph.h" #include "jemalloc/internal/rtree.h" @@ -27,12 +27,12 @@ extern size_t opt_lg_extent_max_active_fit; extern rtree_t extents_rtree; edata_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment, + ecache_t *ecache, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit); void extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - eset_t *eset, edata_t *edata); + ecache_t *ecache, edata_t *edata); edata_t *extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - eset_t *eset, size_t npages_min); + ecache_t *ecache, size_t npages_min); edata_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit); diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index 23312d3b..7b2e84a9 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -44,6 +44,7 @@ + diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj index 76c16c5a..338962b3 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -44,6 +44,7 @@ + diff --git a/src/arena.c b/src/arena.c index a23419a3..2652207a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -56,7 +56,7 @@ static unsigned huge_arena_ind; */ static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, - arena_decay_t *decay, eset_t *eset, bool all, size_t npages_limit, + arena_decay_t *decay, ecache_t *ecache, bool all, size_t npages_limit, size_t npages_decay_max, bool is_background_thread); static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all); @@ -76,8 +76,8 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, *dirty_decay_ms = arena_dirty_decay_ms_get(arena); *muzzy_decay_ms = arena_muzzy_decay_ms_get(arena); *nactive += atomic_load_zu(&arena->nactive, ATOMIC_RELAXED); - *ndirty += eset_npages_get(&arena->eset_dirty); - *nmuzzy += eset_npages_get(&arena->eset_muzzy); + *ndirty += ecache_npages_get(&arena->ecache_dirty); + *nmuzzy += ecache_npages_get(&arena->ecache_muzzy); } void @@ -100,7 +100,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_zu(&astats->mapped, base_mapped + arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped)); arena_stats_accum_zu(&astats->retained, - eset_npages_get(&arena->eset_retained) << LG_PAGE); + ecache_npages_get(&arena->ecache_retained) << LG_PAGE); atomic_store_zu(&astats->edata_avail, atomic_load_zu(&arena->edata_cache.count, ATOMIC_RELAXED), @@ -131,8 +131,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_zu(&astats->metadata_thp, metadata_thp); arena_stats_accum_zu(&astats->resident, base_resident + (((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) + - eset_npages_get(&arena->eset_dirty) + - eset_npages_get(&arena->eset_muzzy)) << LG_PAGE))); + ecache_npages_get(&arena->ecache_dirty) + + ecache_npages_get(&arena->ecache_muzzy)) << LG_PAGE))); arena_stats_accum_zu(&astats->abandoned_vm, atomic_load_zu( &arena->stats.abandoned_vm, ATOMIC_RELAXED)); @@ -174,12 +174,12 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, for (pszind_t i = 0; i < SC_NPSIZES; i++) { size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes, retained_bytes; - dirty = eset_nextents_get(&arena->eset_dirty, i); - muzzy = eset_nextents_get(&arena->eset_muzzy, i); - retained = eset_nextents_get(&arena->eset_retained, i); - dirty_bytes = eset_nbytes_get(&arena->eset_dirty, i); - muzzy_bytes = eset_nbytes_get(&arena->eset_muzzy, i); - retained_bytes = eset_nbytes_get(&arena->eset_retained, i); + dirty = ecache_nextents_get(&arena->ecache_dirty, i); + muzzy = ecache_nextents_get(&arena->ecache_muzzy, i); + retained = ecache_nextents_get(&arena->ecache_retained, i); + dirty_bytes = ecache_nbytes_get(&arena->ecache_dirty, i); + muzzy_bytes = ecache_nbytes_get(&arena->ecache_muzzy, i); + retained_bytes = ecache_nbytes_get(&arena->ecache_retained, i); atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED); atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED); @@ -226,11 +226,11 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large); READ_ARENA_MUTEX_PROF_DATA(edata_cache.mtx, arena_prof_mutex_extent_avail) - READ_ARENA_MUTEX_PROF_DATA(eset_dirty.mtx, + READ_ARENA_MUTEX_PROF_DATA(ecache_dirty.mtx, arena_prof_mutex_extents_dirty) - READ_ARENA_MUTEX_PROF_DATA(eset_muzzy.mtx, + READ_ARENA_MUTEX_PROF_DATA(ecache_muzzy.mtx, arena_prof_mutex_extents_muzzy) - READ_ARENA_MUTEX_PROF_DATA(eset_retained.mtx, + READ_ARENA_MUTEX_PROF_DATA(ecache_retained.mtx, arena_prof_mutex_extents_retained) READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx, arena_prof_mutex_decay_dirty) @@ -258,7 +258,7 @@ arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - extents_dalloc(tsdn, arena, ehooks, &arena->eset_dirty, edata); + extents_dalloc(tsdn, arena, ehooks, &arena->ecache_dirty, edata); if (arena_dirty_decay_ms_get(arena) == 0) { arena_decay_dirty(tsdn, arena, false, true); } else { @@ -434,10 +434,11 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, szind_t szind = sz_size2index(usize); size_t mapped_add; bool commit = true; - edata_t *edata = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty, - NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); + edata_t *edata = extents_alloc(tsdn, arena, ehooks, + &arena->ecache_dirty, NULL, usize, sz_large_pad, alignment, false, + szind, zero, &commit); if (edata == NULL && arena_may_have_muzzy(arena)) { - edata = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy, + edata = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); } @@ -606,10 +607,10 @@ arena_decay_backlog_update(arena_decay_t *decay, uint64_t nadvance_u64, static void arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, - eset_t *eset, size_t current_npages, size_t npages_limit, + ecache_t *ecache, size_t current_npages, size_t npages_limit, bool is_background_thread) { if (current_npages > npages_limit) { - arena_decay_to_limit(tsdn, arena, decay, eset, false, + arena_decay_to_limit(tsdn, arena, decay, ecache, false, npages_limit, current_npages - npages_limit, is_background_thread); } @@ -641,8 +642,8 @@ arena_decay_epoch_advance_helper(arena_decay_t *decay, const nstime_t *time, static void arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, - eset_t *eset, const nstime_t *time, bool is_background_thread) { - size_t current_npages = eset_npages_get(eset); + ecache_t *ecache, const nstime_t *time, bool is_background_thread) { + size_t current_npages = ecache_npages_get(ecache); arena_decay_epoch_advance_helper(decay, time, current_npages); size_t npages_limit = arena_decay_backlog_npages_limit(decay); @@ -651,7 +652,7 @@ arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, current_npages; if (!background_thread_enabled() || is_background_thread) { - arena_decay_try_purge(tsdn, arena, decay, eset, + arena_decay_try_purge(tsdn, arena, decay, ecache, current_npages, npages_limit, is_background_thread); } } @@ -708,15 +709,15 @@ arena_decay_ms_valid(ssize_t decay_ms) { static bool arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, - eset_t *eset, bool is_background_thread) { + ecache_t *ecache, bool is_background_thread) { malloc_mutex_assert_owner(tsdn, &decay->mtx); /* Purge all or nothing if the option is disabled. */ ssize_t decay_ms = arena_decay_ms_read(decay); if (decay_ms <= 0) { if (decay_ms == 0) { - arena_decay_to_limit(tsdn, arena, decay, eset, false, - 0, eset_npages_get(eset), + arena_decay_to_limit(tsdn, arena, decay, ecache, false, + 0, ecache_npages_get(ecache), is_background_thread); } return false; @@ -751,11 +752,11 @@ arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, */ bool advance_epoch = arena_decay_deadline_reached(decay, &time); if (advance_epoch) { - arena_decay_epoch_advance(tsdn, arena, decay, eset, &time, + arena_decay_epoch_advance(tsdn, arena, decay, ecache, &time, is_background_thread); } else if (is_background_thread) { - arena_decay_try_purge(tsdn, arena, decay, eset, - eset_npages_get(eset), + arena_decay_try_purge(tsdn, arena, decay, ecache, + ecache_npages_get(ecache), arena_decay_backlog_npages_limit(decay), is_background_thread); } @@ -780,7 +781,7 @@ arena_muzzy_decay_ms_get(arena_t *arena) { static bool arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, - eset_t *eset, ssize_t decay_ms) { + ecache_t *ecache, ssize_t decay_ms) { if (!arena_decay_ms_valid(decay_ms)) { return true; } @@ -795,7 +796,7 @@ arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, * arbitrary change during initial arena configuration. */ arena_decay_reinit(decay, decay_ms); - arena_maybe_decay(tsdn, arena, decay, eset, false); + arena_maybe_decay(tsdn, arena, decay, ecache, false); malloc_mutex_unlock(tsdn, &decay->mtx); return false; @@ -805,19 +806,19 @@ bool arena_dirty_decay_ms_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_ms) { return arena_decay_ms_set(tsdn, arena, &arena->decay_dirty, - &arena->eset_dirty, decay_ms); + &arena->ecache_dirty, decay_ms); } bool arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_ms) { return arena_decay_ms_set(tsdn, arena, &arena->decay_muzzy, - &arena->eset_muzzy, decay_ms); + &arena->ecache_muzzy, decay_ms); } static size_t arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, - ehooks_t *ehooks, eset_t *eset, size_t npages_limit, + ehooks_t *ehooks, ecache_t *ecache, size_t npages_limit, size_t npages_decay_max, edata_list_t *decay_extents) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -826,7 +827,7 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, size_t nstashed = 0; edata_t *edata; while (nstashed < npages_decay_max && - (edata = extents_evict(tsdn, arena, ehooks, eset, npages_limit)) + (edata = extents_evict(tsdn, arena, ehooks, ecache, npages_limit)) != NULL) { edata_list_append(decay_extents, edata); nstashed += edata_size_get(edata) >> LG_PAGE; @@ -836,8 +837,8 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, static size_t arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - arena_decay_t *decay, eset_t *eset, bool all, edata_list_t *decay_extents, - bool is_background_thread) { + arena_decay_t *decay, ecache_t *ecache, bool all, + edata_list_t *decay_extents, bool is_background_thread) { size_t nmadvise, nunmapped; size_t npurged; @@ -856,7 +857,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, size_t npages = edata_size_get(edata) >> LG_PAGE; npurged += npages; edata_list_remove(decay_extents, edata); - switch (eset_state_get(eset)) { + switch (eset_state_get(&ecache->eset)) { case extent_state_active: not_reached(); case extent_state_dirty: @@ -864,7 +865,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, !extent_purge_lazy_wrapper(tsdn, arena, ehooks, edata, 0, edata_size_get(edata))) { extents_dalloc(tsdn, arena, ehooks, - &arena->eset_muzzy, edata); + &arena->ecache_muzzy, edata); arena_background_thread_inactivity_check(tsdn, arena, is_background_thread); break; @@ -900,14 +901,14 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, /* * npages_limit: Decay at most npages_decay_max pages without violating the - * invariant: (eset_npages_get(extents) >= npages_limit). We need an upper + * invariant: (ecache_npages_get(ecache) >= npages_limit). We need an upper * bound on number of pages in order to prevent unbounded growth (namely in * stashed), otherwise unbounded new pages could be added to extents during the * current decay run, so that the purging thread never finishes. */ static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, - eset_t *eset, bool all, size_t npages_limit, size_t npages_decay_max, + ecache_t *ecache, bool all, size_t npages_limit, size_t npages_decay_max, bool is_background_thread) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 1); @@ -924,11 +925,11 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, edata_list_t decay_extents; edata_list_init(&decay_extents); - size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, eset, + size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, ecache, npages_limit, npages_decay_max, &decay_extents); if (npurge != 0) { size_t npurged = arena_decay_stashed(tsdn, arena, ehooks, decay, - eset, all, &decay_extents, is_background_thread); + ecache, all, &decay_extents, is_background_thread); assert(npurged == npurge); } @@ -938,11 +939,11 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, static bool arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, - eset_t *eset, bool is_background_thread, bool all) { + ecache_t *ecache, bool is_background_thread, bool all) { if (all) { malloc_mutex_lock(tsdn, &decay->mtx); - arena_decay_to_limit(tsdn, arena, decay, eset, all, 0, - eset_npages_get(eset), is_background_thread); + arena_decay_to_limit(tsdn, arena, decay, ecache, all, 0, + ecache_npages_get(ecache), is_background_thread); malloc_mutex_unlock(tsdn, &decay->mtx); return false; @@ -953,7 +954,7 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, return true; } - bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, eset, + bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, ecache, is_background_thread); size_t npages_new; if (epoch_advanced) { @@ -975,18 +976,18 @@ static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) { return arena_decay_impl(tsdn, arena, &arena->decay_dirty, - &arena->eset_dirty, is_background_thread, all); + &arena->ecache_dirty, is_background_thread, all); } static bool arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) { - if (eset_npages_get(&arena->eset_muzzy) == 0 && + if (ecache_npages_get(&arena->ecache_muzzy) == 0 && arena_muzzy_decay_ms_get(arena) <= 0) { return false; } return arena_decay_impl(tsdn, arena, &arena->decay_muzzy, - &arena->eset_muzzy, is_background_thread, all); + &arena->ecache_muzzy, is_background_thread, all); } void @@ -1157,7 +1158,7 @@ arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) { ehooks_t *ehooks = arena_get_ehooks(arena); edata_t *edata; while ((edata = extents_evict(tsdn, arena, ehooks, - &arena->eset_retained, 0)) != NULL) { + &arena->ecache_retained, 0)) != NULL) { extent_destroy_wrapper(tsdn, arena, ehooks, edata); } } @@ -1173,8 +1174,8 @@ arena_destroy(tsd_t *tsd, arena_t *arena) { * Furthermore, the caller (arena_i_destroy_ctl()) purged all cached * extents, so only retained extents may remain. */ - assert(eset_npages_get(&arena->eset_dirty) == 0); - assert(eset_npages_get(&arena->eset_muzzy) == 0); + assert(ecache_npages_get(&arena->ecache_dirty) == 0); + assert(ecache_npages_get(&arena->ecache_muzzy) == 0); /* Deallocate retained memory. */ arena_destroy_retained(tsd_tsdn(tsd), arena); @@ -1230,10 +1231,10 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard szind_t szind = sz_size2index(bin_info->reg_size); bool zero = false; bool commit = true; - edata_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty, + edata_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->ecache_dirty, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); if (slab == NULL && arena_may_have_muzzy(arena)) { - slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy, + slab = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); } @@ -1917,14 +1918,14 @@ arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, } } - malloc_mutex_lock(tsd_tsdn(tsd), &arena->extent_grow_mtx); + malloc_mutex_lock(tsd_tsdn(tsd), &arena->ecache_grow.mtx); if (old_limit != NULL) { - *old_limit = sz_pind2sz(arena->retain_grow_limit); + *old_limit = sz_pind2sz(arena->ecache_grow.limit); } if (new_limit != NULL) { - arena->retain_grow_limit = new_ind; + arena->ecache_grow.limit = new_ind; } - malloc_mutex_unlock(tsd_tsdn(tsd), &arena->extent_grow_mtx); + malloc_mutex_unlock(tsd_tsdn(tsd), &arena->ecache_grow.mtx); return false; } @@ -2016,14 +2017,14 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { * are likely to be reused soon after deallocation, and the cost of * merging/splitting extents is non-trivial. */ - if (eset_init(tsdn, &arena->eset_dirty, extent_state_dirty, true)) { + if (ecache_init(tsdn, &arena->ecache_dirty, extent_state_dirty, true)) { goto label_error; } /* * Coalesce muzzy extents immediately, because operations on them are in * the critical path much less often than for dirty extents. */ - if (eset_init(tsdn, &arena->eset_muzzy, extent_state_muzzy, false)) { + if (ecache_init(tsdn, &arena->ecache_muzzy, extent_state_muzzy, false)) { goto label_error; } /* @@ -2032,7 +2033,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { * coalescing), but also because operations on retained extents are not * in the critical path. */ - if (eset_init(tsdn, &arena->eset_retained, extent_state_retained, + if (ecache_init(tsdn, &arena->ecache_retained, extent_state_retained, false)) { goto label_error; } @@ -2046,10 +2047,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { goto label_error; } - arena->extent_grow_next = sz_psz2ind(HUGEPAGE); - arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS); - if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow", - WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { + if (ecache_grow_init(tsdn, &arena->ecache_grow)) { goto label_error; } @@ -2187,14 +2185,14 @@ arena_prefork1(tsdn_t *tsdn, arena_t *arena) { void arena_prefork2(tsdn_t *tsdn, arena_t *arena) { - malloc_mutex_prefork(tsdn, &arena->extent_grow_mtx); + ecache_grow_prefork(tsdn, &arena->ecache_grow); } void arena_prefork3(tsdn_t *tsdn, arena_t *arena) { - eset_prefork(tsdn, &arena->eset_dirty); - eset_prefork(tsdn, &arena->eset_muzzy); - eset_prefork(tsdn, &arena->eset_retained); + ecache_prefork(tsdn, &arena->ecache_dirty); + ecache_prefork(tsdn, &arena->ecache_muzzy); + ecache_prefork(tsdn, &arena->ecache_retained); } void @@ -2234,10 +2232,10 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); base_postfork_parent(tsdn, arena->base); edata_cache_postfork_parent(tsdn, &arena->edata_cache); - eset_postfork_parent(tsdn, &arena->eset_dirty); - eset_postfork_parent(tsdn, &arena->eset_muzzy); - eset_postfork_parent(tsdn, &arena->eset_retained); - malloc_mutex_postfork_parent(tsdn, &arena->extent_grow_mtx); + ecache_postfork_parent(tsdn, &arena->ecache_dirty); + ecache_postfork_parent(tsdn, &arena->ecache_muzzy); + ecache_postfork_parent(tsdn, &arena->ecache_retained); + ecache_grow_postfork_parent(tsdn, &arena->ecache_grow); malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx); malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx); if (config_stats) { @@ -2280,10 +2278,10 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_postfork_child(tsdn, &arena->large_mtx); base_postfork_child(tsdn, arena->base); edata_cache_postfork_child(tsdn, &arena->edata_cache); - eset_postfork_child(tsdn, &arena->eset_dirty); - eset_postfork_child(tsdn, &arena->eset_muzzy); - eset_postfork_child(tsdn, &arena->eset_retained); - malloc_mutex_postfork_child(tsdn, &arena->extent_grow_mtx); + ecache_postfork_child(tsdn, &arena->ecache_dirty); + ecache_postfork_child(tsdn, &arena->ecache_muzzy); + ecache_postfork_child(tsdn, &arena->ecache_retained); + ecache_grow_postfork_child(tsdn, &arena->ecache_grow); malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx); malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx); if (config_stats) { diff --git a/src/background_thread.c b/src/background_thread.c index 400dae53..90b027ed 100644 --- a/src/background_thread.c +++ b/src/background_thread.c @@ -114,7 +114,7 @@ decay_npurge_after_interval(arena_decay_t *decay, size_t interval) { static uint64_t arena_decay_compute_purge_interval_impl(tsdn_t *tsdn, arena_decay_t *decay, - eset_t *eset) { + ecache_t *ecache) { if (malloc_mutex_trylock(tsdn, &decay->mtx)) { /* Use minimal interval if decay is contended. */ return BACKGROUND_THREAD_MIN_INTERVAL_NS; @@ -130,7 +130,7 @@ arena_decay_compute_purge_interval_impl(tsdn_t *tsdn, arena_decay_t *decay, uint64_t decay_interval_ns = nstime_ns(&decay->interval); assert(decay_interval_ns > 0); - size_t npages = eset_npages_get(eset); + size_t npages = ecache_npages_get(ecache); if (npages == 0) { unsigned i; for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) { @@ -202,12 +202,12 @@ static uint64_t arena_decay_compute_purge_interval(tsdn_t *tsdn, arena_t *arena) { uint64_t i1, i2; i1 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_dirty, - &arena->eset_dirty); + &arena->ecache_dirty); if (i1 == BACKGROUND_THREAD_MIN_INTERVAL_NS) { return i1; } i2 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_muzzy, - &arena->eset_muzzy); + &arena->ecache_muzzy); return i1 < i2 ? i1 : i2; } @@ -717,8 +717,8 @@ background_thread_interval_check(tsdn_t *tsdn, arena_t *arena, if (info->npages_to_purge_new > BACKGROUND_THREAD_NPAGES_THRESHOLD) { should_signal = true; } else if (unlikely(background_thread_indefinite_sleep(info)) && - (eset_npages_get(&arena->eset_dirty) > 0 || - eset_npages_get(&arena->eset_muzzy) > 0 || + (ecache_npages_get(&arena->ecache_dirty) > 0 || + ecache_npages_get(&arena->ecache_muzzy) > 0 || info->npages_to_purge_new > 0)) { should_signal = true; } else { diff --git a/src/ctl.c b/src/ctl.c index a58b22ba..eee12770 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -3011,9 +3011,9 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, } MUTEX_PROF_RESET(arena->large_mtx); MUTEX_PROF_RESET(arena->edata_cache.mtx); - MUTEX_PROF_RESET(arena->eset_dirty.mtx); - MUTEX_PROF_RESET(arena->eset_muzzy.mtx); - MUTEX_PROF_RESET(arena->eset_retained.mtx); + MUTEX_PROF_RESET(arena->ecache_dirty.mtx); + MUTEX_PROF_RESET(arena->ecache_muzzy.mtx); + MUTEX_PROF_RESET(arena->ecache_retained.mtx); MUTEX_PROF_RESET(arena->decay_dirty.mtx); MUTEX_PROF_RESET(arena->decay_muzzy.mtx); MUTEX_PROF_RESET(arena->tcache_ql_mtx); diff --git a/src/ecache.c b/src/ecache.c new file mode 100644 index 00000000..7155de35 --- /dev/null +++ b/src/ecache.c @@ -0,0 +1,54 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +bool +ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, + bool delay_coalesce) { + if (malloc_mutex_init(&ecache->mtx, "extents", WITNESS_RANK_EXTENTS, + malloc_mutex_rank_exclusive)) { + return true; + } + eset_init(&ecache->eset, state, delay_coalesce); + return false; +} + +void +ecache_prefork(tsdn_t *tsdn, ecache_t *ecache) { + malloc_mutex_prefork(tsdn, &ecache->mtx); +} + +void +ecache_postfork_parent(tsdn_t *tsdn, ecache_t *ecache) { + malloc_mutex_postfork_parent(tsdn, &ecache->mtx); +} + +void +ecache_postfork_child(tsdn_t *tsdn, ecache_t *ecache) { + malloc_mutex_postfork_child(tsdn, &ecache->mtx); +} + +bool +ecache_grow_init(tsdn_t *tsdn, ecache_grow_t *ecache_grow) { + ecache_grow->next = sz_psz2ind(HUGEPAGE); + ecache_grow->limit = sz_psz2ind(SC_LARGE_MAXCLASS); + if (malloc_mutex_init(&ecache_grow->mtx, "extent_grow", + WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { + return true; + } + return false; +} + +void +ecache_grow_prefork(tsdn_t *tsdn, ecache_grow_t *ecache_grow) { + malloc_mutex_prefork(tsdn, &ecache_grow->mtx); +} + +void +ecache_grow_postfork_parent(tsdn_t *tsdn, ecache_grow_t *ecache_grow) { + malloc_mutex_postfork_parent(tsdn, &ecache_grow->mtx); +} + +void +ecache_grow_postfork_child(tsdn_t *tsdn, ecache_grow_t *ecache_grow) { + malloc_mutex_postfork_child(tsdn, &ecache_grow->mtx); +} diff --git a/src/eset.c b/src/eset.c index 88b9c8c7..e71868c8 100644 --- a/src/eset.c +++ b/src/eset.c @@ -8,13 +8,9 @@ const bitmap_info_t eset_bitmap_info = BITMAP_INFO_INITIALIZER(SC_NPSIZES+1); -bool -eset_init(tsdn_t *tsdn, eset_t *eset, extent_state_t state, +void +eset_init(eset_t *eset, extent_state_t state, bool delay_coalesce) { - if (malloc_mutex_init(&eset->mtx, "extents", WITNESS_RANK_EXTENTS, - malloc_mutex_rank_exclusive)) { - return true; - } for (unsigned i = 0; i < SC_NPSIZES + 1; i++) { edata_heap_new(&eset->heaps[i]); } @@ -23,7 +19,6 @@ eset_init(tsdn_t *tsdn, eset_t *eset, extent_state_t state, atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED); eset->state = state; eset->delay_coalesce = delay_coalesce; - return false; } extent_state_t @@ -63,8 +58,7 @@ eset_stats_sub(eset_t *eset, pszind_t pind, size_t sz) { } void -eset_insert_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) { - malloc_mutex_assert_owner(tsdn, &eset->mtx); +eset_insert(eset_t *eset, edata_t *edata) { assert(edata_state_get(edata) == eset->state); size_t size = edata_size_get(edata); @@ -94,8 +88,7 @@ eset_insert_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) { } void -eset_remove_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) { - malloc_mutex_assert_owner(tsdn, &eset->mtx); +eset_remove(eset_t *eset, edata_t *edata) { assert(edata_state_get(edata) == eset->state); size_t size = edata_size_get(edata); @@ -114,9 +107,13 @@ eset_remove_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) { edata_list_remove(&eset->lru, edata); size_t npages = size >> LG_PAGE; /* - * As in eset_insert_locked, we hold eset->mtx and so don't need atomic + * As in eset_insert, we hold eset->mtx and so don't need atomic * operations for updating eset->npages. */ + /* + * This class is not thread-safe in general; we rely on external + * synchronization for all mutating operations. + */ size_t cur_extents_npages = atomic_load_zu(&eset->npages, ATOMIC_RELAXED); assert(cur_extents_npages >= npages); @@ -166,7 +163,7 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size, * large enough. */ static edata_t * -eset_first_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t size) { +eset_first_fit(eset_t *eset, size_t size) { edata_t *ret = NULL; pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(size)); @@ -211,16 +208,14 @@ eset_first_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t size) { } edata_t * -eset_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t esize, size_t alignment) { - malloc_mutex_assert_owner(tsdn, &eset->mtx); - +eset_fit(eset_t *eset, size_t esize, size_t alignment) { size_t max_size = esize + PAGE_CEILING(alignment) - PAGE; /* Beware size_t wrap-around. */ if (max_size < esize) { return NULL; } - edata_t *edata = eset_first_fit_locked(tsdn, eset, max_size); + edata_t *edata = eset_first_fit(eset, max_size); if (alignment > PAGE && edata == NULL) { /* @@ -233,18 +228,3 @@ eset_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t esize, size_t alignment) { return edata; } - -void -eset_prefork(tsdn_t *tsdn, eset_t *eset) { - malloc_mutex_prefork(tsdn, &eset->mtx); -} - -void -eset_postfork_parent(tsdn_t *tsdn, eset_t *eset) { - malloc_mutex_postfork_parent(tsdn, &eset->mtx); -} - -void -eset_postfork_child(tsdn_t *tsdn, eset_t *eset) { - malloc_mutex_postfork_child(tsdn, &eset->mtx); -} diff --git a/src/extent2.c b/src/extent2.c index 21f9cdbd..3b47e070 100644 --- a/src/extent2.c +++ b/src/extent2.c @@ -45,13 +45,13 @@ static atomic_zu_t highpages; static void extent_deregister(tsdn_t *tsdn, edata_t *edata); static edata_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - eset_t *eset, void *new_addr, size_t usize, size_t pad, size_t alignment, + ecache_t *ecache, void *new_addr, size_t usize, size_t pad, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained); static edata_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, - ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, + ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced, bool growing_retained); static void extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - eset_t *eset, edata_t *edata, bool growing_retained); + ecache_t *ecache, edata_t *edata, bool growing_retained); /******************************************************************************/ @@ -165,22 +165,22 @@ extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata, static bool extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata) { + rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata) { edata_state_set(edata, extent_state_active); bool coalesced; - edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, eset, + edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, ecache, edata, &coalesced, false); - edata_state_set(edata, eset_state_get(eset)); + edata_state_set(edata, eset_state_get(&ecache->eset)); if (!coalesced) { return true; } - eset_insert_locked(tsdn, eset, edata); + eset_insert(&ecache->eset, edata); return false; } edata_t * -extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, +extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { assert(size + pad != 0); @@ -188,14 +188,14 @@ extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - edata_t *edata = extent_recycle(tsdn, arena, ehooks, eset, new_addr, + edata_t *edata = extent_recycle(tsdn, arena, ehooks, ecache, new_addr, size, pad, alignment, slab, szind, zero, commit, false); assert(edata == NULL || edata_dumpable_get(edata)); return edata; } void -extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, +extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata) { assert(edata_base_get(edata) != NULL); assert(edata_size_get(edata) != 0); @@ -206,16 +206,16 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, edata_addr_set(edata, edata_base_get(edata)); edata_zeroed_set(edata, false); - extent_record(tsdn, arena, ehooks, eset, edata, false); + extent_record(tsdn, arena, ehooks, ecache, edata, false); } edata_t * -extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, +extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache, size_t npages_min) { rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - malloc_mutex_lock(tsdn, &eset->mtx); + malloc_mutex_lock(tsdn, &ecache->mtx); /* * Get the LRU coalesced extent, if any. If coalescing was delayed, @@ -224,24 +224,23 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, edata_t *edata; while (true) { /* Get the LRU extent, if any. */ - edata = edata_list_first(&eset->lru); + edata = edata_list_first(&ecache->eset.lru); if (edata == NULL) { goto label_return; } /* Check the eviction limit. */ - size_t extents_npages = atomic_load_zu(&eset->npages, - ATOMIC_RELAXED); + size_t extents_npages = ecache_npages_get(ecache); if (extents_npages <= npages_min) { edata = NULL; goto label_return; } - eset_remove_locked(tsdn, eset, edata); - if (!eset->delay_coalesce) { + eset_remove(&ecache->eset, edata); + if (!ecache->eset.delay_coalesce) { break; } /* Try to coalesce. */ if (extent_try_delayed_coalesce(tsdn, arena, ehooks, rtree_ctx, - eset, edata)) { + ecache, edata)) { break; } /* @@ -254,7 +253,7 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, * Either mark the extent active or deregister it to protect against * concurrent operations. */ - switch (eset_state_get(eset)) { + switch (eset_state_get(&ecache->eset)) { case extent_state_active: not_reached(); case extent_state_dirty: @@ -269,7 +268,7 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, } label_return: - malloc_mutex_unlock(tsdn, &eset->mtx); + malloc_mutex_unlock(tsdn, &ecache->mtx); return edata; } @@ -278,8 +277,8 @@ label_return: * indicates OOM), e.g. when trying to split an existing extent. */ static void -extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, - edata_t *edata, bool growing_retained) { +extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + ecache_t *ecache, edata_t *edata, bool growing_retained) { size_t sz = edata_size_get(edata); if (config_stats) { arena_stats_accum_zu(&arena->stats.abandoned_vm, sz); @@ -288,7 +287,7 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, * Leak extent after making sure its pages have already been purged, so * that this is only a virtual memory leak. */ - if (eset_state_get(eset) == extent_state_dirty) { + if (eset_state_get(&ecache->eset) == extent_state_dirty) { if (extent_purge_lazy_impl(tsdn, arena, ehooks, edata, 0, sz, growing_retained)) { extent_purge_forced_impl(tsdn, arena, ehooks, edata, 0, @@ -299,30 +298,30 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, } static void -extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset, +extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache, edata_t *edata) { assert(edata_arena_ind_get(edata) == arena_ind_get(arena)); assert(edata_state_get(edata) == extent_state_active); - edata_state_set(edata, eset_state_get(eset)); - eset_insert_locked(tsdn, eset, edata); + edata_state_set(edata, eset_state_get(&ecache->eset)); + eset_insert(&ecache->eset, edata); } static void -extent_deactivate(tsdn_t *tsdn, arena_t *arena, eset_t *eset, +extent_deactivate(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache, edata_t *edata) { - malloc_mutex_lock(tsdn, &eset->mtx); - extent_deactivate_locked(tsdn, arena, eset, edata); - malloc_mutex_unlock(tsdn, &eset->mtx); + malloc_mutex_lock(tsdn, &ecache->mtx); + extent_deactivate_locked(tsdn, arena, ecache, edata); + malloc_mutex_unlock(tsdn, &ecache->mtx); } static void -extent_activate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset, +extent_activate_locked(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache, edata_t *edata) { assert(edata_arena_ind_get(edata) == arena_ind_get(arena)); - assert(edata_state_get(edata) == eset_state_get(eset)); + assert(edata_state_get(edata) == eset_state_get(&ecache->eset)); - eset_remove_locked(tsdn, eset, edata); + eset_remove(&ecache->eset, edata); edata_state_set(edata, extent_state_active); } @@ -515,12 +514,12 @@ extent_deregister_no_gdump_sub(tsdn_t *tsdn, edata_t *edata) { } /* - * Tries to find and remove an extent from eset that can be used for the + * Tries to find and remove an extent from ecache that can be used for the * given allocation request. */ static edata_t * extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size, + rtree_ctx_t *rtree_ctx, ecache_t *ecache, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); @@ -543,7 +542,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, } size_t esize = size + pad; - malloc_mutex_lock(tsdn, &eset->mtx); + malloc_mutex_lock(tsdn, &ecache->mtx); edata_t *edata; if (new_addr != NULL) { edata = extent_lock_edata_from_addr(tsdn, rtree_ctx, new_addr, @@ -557,21 +556,22 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, assert(edata_base_get(edata) == new_addr); if (edata_arena_ind_get(edata) != arena_ind_get(arena) || edata_size_get(edata) < esize - || edata_state_get(edata) != eset_state_get(eset)) { + || edata_state_get(edata) + != eset_state_get(&ecache->eset)) { edata = NULL; } extent_unlock_edata(tsdn, unlock_edata); } } else { - edata = eset_fit_locked(tsdn, eset, esize, alignment); + edata = eset_fit(&ecache->eset, esize, alignment); } if (edata == NULL) { - malloc_mutex_unlock(tsdn, &eset->mtx); + malloc_mutex_unlock(tsdn, &ecache->mtx); return NULL; } - extent_activate_locked(tsdn, arena, eset, edata); - malloc_mutex_unlock(tsdn, &eset->mtx); + extent_activate_locked(tsdn, arena, ecache, edata); + malloc_mutex_unlock(tsdn, &ecache->mtx); return edata; } @@ -580,7 +580,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, * Given an allocation request and an extent guaranteed to be able to satisfy * it, this splits off lead and trail extents, leaving edata pointing to an * extent satisfying the allocation. - * This function doesn't put lead or trail into any eset_t; it's the caller's + * This function doesn't put lead or trail into any ecache; it's the caller's * job to ensure that they can be reused. */ typedef enum { @@ -676,11 +676,11 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, * This fulfills the indicated allocation request out of the given extent (which * the caller should have ensured was big enough). If there's any unused space * before or after the resulting allocation, that space is given its own extent - * and put back into eset. + * and put back into ecache. */ static edata_t * extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size, + rtree_ctx_t *rtree_ctx, ecache_t *ecache, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, edata_t *edata, bool growing_retained) { edata_t *lead; @@ -697,19 +697,19 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, && !opt_retain) { /* * Split isn't supported (implies Windows w/o retain). Avoid - * leaking the eset. + * leaking the extent. */ assert(to_leak != NULL && lead == NULL && trail == NULL); - extent_deactivate(tsdn, arena, eset, to_leak); + extent_deactivate(tsdn, arena, ecache, to_leak); return NULL; } if (result == extent_split_interior_ok) { if (lead != NULL) { - extent_deactivate(tsdn, arena, eset, lead); + extent_deactivate(tsdn, arena, ecache, lead); } if (trail != NULL) { - extent_deactivate(tsdn, arena, eset, trail); + extent_deactivate(tsdn, arena, ecache, trail); } return edata; } else { @@ -724,7 +724,7 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, if (to_leak != NULL) { void *leak = edata_base_get(to_leak); extent_deregister_no_gdump_sub(tsdn, to_leak); - extents_abandon_vm(tsdn, arena, ehooks, eset, to_leak, + extents_abandon_vm(tsdn, arena, ehooks, ecache, to_leak, growing_retained); assert(extent_lock_edata_from_addr(tsdn, rtree_ctx, leak, false) == NULL); @@ -736,10 +736,10 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, /* * Tries to satisfy the given allocation request by reusing one of the extents - * in the given eset_t. + * in the given ecache_t. */ static edata_t * -extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, +extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -752,13 +752,13 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); edata_t *edata = extent_recycle_extract(tsdn, arena, ehooks, - rtree_ctx, eset, new_addr, size, pad, alignment, slab, + rtree_ctx, ecache, new_addr, size, pad, alignment, slab, growing_retained); if (edata == NULL) { return NULL; } - edata = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, eset, + edata = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, ecache, new_addr, size, pad, alignment, slab, szind, edata, growing_retained); if (edata == NULL) { @@ -768,7 +768,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, if (*commit && !edata_committed_get(edata)) { if (extent_commit_impl(tsdn, arena, ehooks, edata, 0, edata_size_get(edata), growing_retained)) { - extent_record(tsdn, arena, ehooks, eset, edata, + extent_record(tsdn, arena, ehooks, ecache, edata, growing_retained); return NULL; } @@ -810,7 +810,7 @@ static edata_t * extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { - malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx); + malloc_mutex_assert_owner(tsdn, &arena->ecache_grow.mtx); assert(pad == 0 || !slab); assert(!*zero || !slab); @@ -825,15 +825,15 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, * satisfy this request. */ pszind_t egn_skip = 0; - size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); + size_t alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip); while (alloc_size < alloc_size_min) { egn_skip++; - if (arena->extent_grow_next + egn_skip >= + if (arena->ecache_grow.next + egn_skip >= sz_psz2ind(SC_LARGE_MAXCLASS)) { /* Outside legal range. */ goto label_err; } - alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); + alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip); } edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache, @@ -881,11 +881,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, if (result == extent_split_interior_ok) { if (lead != NULL) { extent_record(tsdn, arena, ehooks, - &arena->eset_retained, lead, true); + &arena->ecache_retained, lead, true); } if (trail != NULL) { extent_record(tsdn, arena, ehooks, - &arena->eset_retained, trail, true); + &arena->ecache_retained, trail, true); } } else { /* @@ -898,12 +898,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_gdump_add(tsdn, to_salvage); } extent_record(tsdn, arena, ehooks, - &arena->eset_retained, to_salvage, true); + &arena->ecache_retained, to_salvage, true); } if (to_leak != NULL) { extent_deregister_no_gdump_sub(tsdn, to_leak); extents_abandon_vm(tsdn, arena, ehooks, - &arena->eset_retained, to_leak, true); + &arena->ecache_retained, to_leak, true); } goto label_err; } @@ -912,7 +912,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, if (extent_commit_impl(tsdn, arena, ehooks, edata, 0, edata_size_get(edata), true)) { extent_record(tsdn, arena, ehooks, - &arena->eset_retained, edata, true); + &arena->ecache_retained, edata, true); goto label_err; } /* A successful commit should return zeroed memory. */ @@ -930,14 +930,14 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, * Increment extent_grow_next if doing so wouldn't exceed the allowed * range. */ - if (arena->extent_grow_next + egn_skip + 1 <= - arena->retain_grow_limit) { - arena->extent_grow_next += egn_skip + 1; + if (arena->ecache_grow.next + egn_skip + 1 <= + arena->ecache_grow.limit) { + arena->ecache_grow.next += egn_skip + 1; } else { - arena->extent_grow_next = arena->retain_grow_limit; + arena->ecache_grow.next = arena->ecache_grow.limit; } /* All opportunities for failure are past. */ - malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); + malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx); if (config_prof) { /* Adjust gdump stats now that extent is final size. */ @@ -962,7 +962,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, return edata; label_err: - malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); + malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx); return NULL; } @@ -973,13 +973,13 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, assert(size != 0); assert(alignment != 0); - malloc_mutex_lock(tsdn, &arena->extent_grow_mtx); + malloc_mutex_lock(tsdn, &arena->ecache_grow.mtx); edata_t *edata = extent_recycle(tsdn, arena, ehooks, - &arena->eset_retained, new_addr, size, pad, alignment, slab, + &arena->ecache_retained, new_addr, size, pad, alignment, slab, szind, zero, commit, true); if (edata != NULL) { - malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); + malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx); if (config_prof) { extent_gdump_add(tsdn, edata); } @@ -988,9 +988,9 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, alignment, slab, szind, zero, commit); /* extent_grow_retained() always releases extent_grow_mtx. */ } else { - malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); + malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx); } - malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx); + malloc_mutex_assert_not_owner(tsdn, &arena->ecache_grow.mtx); return edata; } @@ -1054,7 +1054,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, } static bool -extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner, +extent_can_coalesce(arena_t *arena, ecache_t *ecache, const edata_t *inner, const edata_t *outer) { assert(edata_arena_ind_get(inner) == arena_ind_get(arena)); if (edata_arena_ind_get(outer) != arena_ind_get(arena)) { @@ -1062,7 +1062,7 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner, } assert(edata_state_get(inner) == extent_state_active); - if (edata_state_get(outer) != eset->state) { + if (edata_state_get(outer) != ecache->eset.state) { return false; } @@ -1074,19 +1074,20 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner, } static bool -extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, - edata_t *inner, edata_t *outer, bool forward, bool growing_retained) { - assert(extent_can_coalesce(arena, eset, inner, outer)); +extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + ecache_t *ecache, edata_t *inner, edata_t *outer, bool forward, + bool growing_retained) { + assert(extent_can_coalesce(arena, ecache, inner, outer)); - extent_activate_locked(tsdn, arena, eset, outer); + extent_activate_locked(tsdn, arena, ecache, outer); - malloc_mutex_unlock(tsdn, &eset->mtx); + malloc_mutex_unlock(tsdn, &ecache->mtx); bool err = extent_merge_impl(tsdn, arena, ehooks, forward ? inner : outer, forward ? outer : inner, growing_retained); - malloc_mutex_lock(tsdn, &eset->mtx); + malloc_mutex_lock(tsdn, &ecache->mtx); if (err) { - extent_deactivate_locked(tsdn, arena, eset, outer); + extent_deactivate_locked(tsdn, arena, ecache, outer); } return err; @@ -1094,7 +1095,7 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, static edata_t * extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced, + rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced, bool growing_retained, bool inactive_only) { /* * We avoid checking / locking inactive neighbors for large size @@ -1114,19 +1115,19 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_past_get(edata), inactive_only); if (next != NULL) { /* - * eset->mtx only protects against races for - * like-state eset, so call extent_can_coalesce() + * ecache->mtx only protects against races for + * like-state extents, so call extent_can_coalesce() * before releasing next's pool lock. */ - bool can_coalesce = extent_can_coalesce(arena, eset, + bool can_coalesce = extent_can_coalesce(arena, ecache, edata, next); extent_unlock_edata(tsdn, next); if (can_coalesce && !extent_coalesce(tsdn, arena, - ehooks, eset, edata, next, true, + ehooks, ecache, edata, next, true, growing_retained)) { - if (eset->delay_coalesce) { + if (ecache->eset.delay_coalesce) { /* Do minimal coalescing. */ *coalesced = true; return edata; @@ -1139,15 +1140,15 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *prev = extent_lock_edata_from_addr(tsdn, rtree_ctx, edata_before_get(edata), inactive_only); if (prev != NULL) { - bool can_coalesce = extent_can_coalesce(arena, eset, + bool can_coalesce = extent_can_coalesce(arena, ecache, edata, prev); extent_unlock_edata(tsdn, prev); if (can_coalesce && !extent_coalesce(tsdn, arena, - ehooks, eset, edata, prev, false, + ehooks, ecache, edata, prev, false, growing_retained)) { edata = prev; - if (eset->delay_coalesce) { + if (ecache->eset.delay_coalesce) { /* Do minimal coalescing. */ *coalesced = true; return edata; @@ -1157,7 +1158,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, } } while (again); - if (eset->delay_coalesce) { + if (ecache->eset.delay_coalesce) { *coalesced = false; } return edata; @@ -1165,35 +1166,35 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, static edata_t * extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced, + rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced, bool growing_retained) { - return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset, + return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, ecache, edata, coalesced, growing_retained, false); } static edata_t * extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, - rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced, + rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced, bool growing_retained) { - return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset, + return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, ecache, edata, coalesced, growing_retained, true); } /* * Does the metadata management portions of putting an unused extent into the - * given eset_t (coalesces, deregisters slab interiors, the heap operations). + * given ecache_t (coalesces, deregisters slab interiors, the heap operations). */ static void -extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, +extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata, bool growing_retained) { rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - assert((eset_state_get(eset) != extent_state_dirty && - eset_state_get(eset) != extent_state_muzzy) || + assert((eset_state_get(&ecache->eset) != extent_state_dirty && + eset_state_get(&ecache->eset) != extent_state_muzzy) || !edata_zeroed_get(edata)); - malloc_mutex_lock(tsdn, &eset->mtx); + malloc_mutex_lock(tsdn, &ecache->mtx); edata_szind_set(edata, SC_NSIZES); if (edata_slab_get(edata)) { @@ -1204,29 +1205,29 @@ extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, assert(rtree_edata_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)edata_base_get(edata), true) == edata); - if (!eset->delay_coalesce) { + if (!ecache->eset.delay_coalesce) { edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, - eset, edata, NULL, growing_retained); + ecache, edata, NULL, growing_retained); } else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) { - assert(eset == &arena->eset_dirty); - /* Always coalesce large eset eagerly. */ + assert(ecache == &arena->ecache_dirty); + /* Always coalesce large extents eagerly. */ bool coalesced; do { assert(edata_state_get(edata) == extent_state_active); edata = extent_try_coalesce_large(tsdn, arena, ehooks, - rtree_ctx, eset, edata, &coalesced, + rtree_ctx, ecache, edata, &coalesced, growing_retained); } while (coalesced); if (edata_size_get(edata) >= oversize_threshold) { /* Shortcut to purge the oversize extent eagerly. */ - malloc_mutex_unlock(tsdn, &eset->mtx); + malloc_mutex_unlock(tsdn, &ecache->mtx); arena_decay_extent(tsdn, arena, ehooks, edata); return; } } - extent_deactivate_locked(tsdn, arena, eset, edata); + extent_deactivate_locked(tsdn, arena, ecache, edata); - malloc_mutex_unlock(tsdn, &eset->mtx); + malloc_mutex_unlock(tsdn, &ecache->mtx); } void @@ -1312,7 +1313,8 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_gdump_sub(tsdn, edata); } - extent_record(tsdn, arena, ehooks, &arena->eset_retained, edata, false); + extent_record(tsdn, arena, ehooks, &arena->ecache_retained, edata, + false); } void diff --git a/src/large.c b/src/large.c index 67b47453..b8b70f4e 100644 --- a/src/large.c +++ b/src/large.c @@ -149,10 +149,10 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize, bool commit = true; edata_t *trail; bool new_mapping; - if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty, + if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->ecache_dirty, edata_past_get(edata), trailsize, 0, CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL - || (trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy, + || (trail = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy, edata_past_get(edata), trailsize, 0, CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) { if (config_stats) { diff --git a/test/unit/retained.c b/test/unit/retained.c index 7993fd3d..e6b6f7b7 100644 --- a/test/unit/retained.c +++ b/test/unit/retained.c @@ -142,7 +142,7 @@ TEST_BEGIN(test_retained) { size_t usable = 0; size_t fragmented = 0; for (pszind_t pind = sz_psz2ind(HUGEPAGE); pind < - arena->extent_grow_next; pind++) { + arena->ecache_grow.next; pind++) { size_t psz = sz_pind2sz(pind); size_t psz_fragmented = psz % esz; size_t psz_usable = psz - psz_fragmented;