diff --git a/Makefile.in b/Makefile.in
index f75ae4b6..71458487 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -104,6 +104,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/ckh.c \
$(srcroot)src/ctl.c \
$(srcroot)src/div.c \
+ $(srcroot)src/ecache.c \
$(srcroot)src/edata.c \
$(srcroot)src/edata_cache.c \
$(srcroot)src/ehooks.c \
diff --git a/include/jemalloc/internal/arena_structs.h b/include/jemalloc/internal/arena_structs.h
index 38c8b27c..48d13b8c 100644
--- a/include/jemalloc/internal/arena_structs.h
+++ b/include/jemalloc/internal/arena_structs.h
@@ -5,8 +5,8 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/bitmap.h"
+#include "jemalloc/internal/ecache.h"
#include "jemalloc/internal/edata_cache.h"
-#include "jemalloc/internal/eset.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
@@ -53,7 +53,7 @@ struct arena_decay_s {
/*
* Number of unpurged pages at beginning of current epoch. During epoch
* advancement we use the delta between arena->decay_*.nunpurged and
- * eset_npages_get(&arena->extents_*) to determine how many dirty pages,
+ * ecache_npages_get(&arena->ecache_*) to determine how many dirty pages,
* if any, were generated.
*/
size_t nunpurged;
@@ -155,9 +155,9 @@ struct arena_s {
*
* Synchronization: internal.
*/
- eset_t eset_dirty;
- eset_t eset_muzzy;
- eset_t eset_retained;
+ ecache_t ecache_dirty;
+ ecache_t ecache_muzzy;
+ ecache_t ecache_retained;
/*
* Decay-based purging state, responsible for scheduling extent state
@@ -168,22 +168,8 @@ struct arena_s {
arena_decay_t decay_dirty; /* dirty --> muzzy */
arena_decay_t decay_muzzy; /* muzzy --> retained */
- /*
- * Next extent size class in a growing series to use when satisfying a
- * request via the extent hooks (only if opt_retain). This limits the
- * number of disjoint virtual memory ranges so that extent merging can
- * be effective even if multiple arenas' extent allocation requests are
- * highly interleaved.
- *
- * retain_grow_limit is the max allowed size ind to expand (unless the
- * required size is greater). Default is no limit, and controlled
- * through mallctl only.
- *
- * Synchronization: extent_grow_mtx
- */
- pszind_t extent_grow_next;
- pszind_t retain_grow_limit;
- malloc_mutex_t extent_grow_mtx;
+ /* The grow info for the retained ecache. */
+ ecache_grow_t ecache_grow;
/* The source of edata_t objects. */
edata_cache_t edata_cache;
diff --git a/include/jemalloc/internal/ecache.h b/include/jemalloc/internal/ecache.h
new file mode 100644
index 00000000..70857203
--- /dev/null
+++ b/include/jemalloc/internal/ecache.h
@@ -0,0 +1,59 @@
+#ifndef JEMALLOC_INTERNAL_ECACHE_H
+#define JEMALLOC_INTERNAL_ECACHE_H
+
+#include "jemalloc/internal/eset.h"
+#include "jemalloc/internal/mutex.h"
+
+typedef struct ecache_s ecache_t;
+struct ecache_s {
+ malloc_mutex_t mtx;
+ eset_t eset;
+};
+
+typedef struct ecache_grow_s ecache_grow_t;
+struct ecache_grow_s {
+ /*
+ * Next extent size class in a growing series to use when satisfying a
+ * request via the extent hooks (only if opt_retain). This limits the
+ * number of disjoint virtual memory ranges so that extent merging can
+ * be effective even if multiple arenas' extent allocation requests are
+ * highly interleaved.
+ *
+ * retain_grow_limit is the max allowed size ind to expand (unless the
+ * required size is greater). Default is no limit, and controlled
+ * through mallctl only.
+ *
+ * Synchronization: extent_grow_mtx
+ */
+ pszind_t next;
+ pszind_t limit;
+ malloc_mutex_t mtx;
+};
+
+static inline size_t
+ecache_npages_get(ecache_t *ecache) {
+ return eset_npages_get(&ecache->eset);
+}
+/* Get the number of extents in the given page size index. */
+static inline size_t
+ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
+ return eset_nextents_get(&ecache->eset, ind);
+}
+/* Get the sum total bytes of the extents in the given page size index. */
+static inline size_t
+ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
+ return eset_nbytes_get(&ecache->eset, ind);
+}
+
+bool ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state,
+ bool delay_coalesce);
+void ecache_prefork(tsdn_t *tsdn, ecache_t *ecache);
+void ecache_postfork_parent(tsdn_t *tsdn, ecache_t *ecache);
+void ecache_postfork_child(tsdn_t *tsdn, ecache_t *ecache);
+
+bool ecache_grow_init(tsdn_t *tsdn, ecache_grow_t *ecache_grow);
+void ecache_grow_prefork(tsdn_t *tsdn, ecache_grow_t *ecache_grow);
+void ecache_grow_postfork_parent(tsdn_t *tsdn, ecache_grow_t *ecache_grow);
+void ecache_grow_postfork_child(tsdn_t *tsdn, ecache_grow_t *ecache_grow);
+
+#endif /* JEMALLOC_INTERNAL_ECACHE_H */
diff --git a/include/jemalloc/internal/eset.h b/include/jemalloc/internal/eset.h
index e76257af..bbc6b5cd 100644
--- a/include/jemalloc/internal/eset.h
+++ b/include/jemalloc/internal/eset.h
@@ -9,42 +9,25 @@
/*
* An eset ("extent set") is a quantized collection of extents, with built-in
* LRU queue.
+ *
+ * This class is not thread-safe; synchronization must be done externally if
+ * there are mutating operations. One exception is the stats counters, which
+ * may be read without any locking.
*/
typedef struct eset_s eset_t;
struct eset_s {
- malloc_mutex_t mtx;
-
- /*
- * Quantized per size class heaps of extents.
- *
- * Synchronization: mtx.
- */
+ /* Quantized per size class heaps of extents. */
edata_heap_t heaps[SC_NPSIZES + 1];
atomic_zu_t nextents[SC_NPSIZES + 1];
atomic_zu_t nbytes[SC_NPSIZES + 1];
- /*
- * Bitmap for which set bits correspond to non-empty heaps.
- *
- * Synchronization: mtx.
- */
+ /* Bitmap for which set bits correspond to non-empty heaps. */
bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)];
- /*
- * LRU of all extents in heaps.
- *
- * Synchronization: mtx.
- */
+ /* LRU of all extents in heaps. */
edata_list_t lru;
- /*
- * Page sum for all extents in heaps.
- *
- * The synchronization here is a little tricky. Modifications to npages
- * must hold mtx, but reads need not (though, a reader who sees npages
- * without holding the mutex can't assume anything about the rest of the
- * state of the eset_t).
- */
+ /* Page sum for all extents in heaps. */
atomic_zu_t npages;
/* All stored extents must be in the same state. */
@@ -57,8 +40,7 @@ struct eset_s {
bool delay_coalesce;
};
-bool eset_init(tsdn_t *tsdn, eset_t *eset, extent_state_t state,
- bool delay_coalesce);
+void eset_init(eset_t *eset, extent_state_t state, bool delay_coalesce);
extent_state_t eset_state_get(const eset_t *eset);
size_t eset_npages_get(eset_t *eset);
@@ -67,17 +49,12 @@ size_t eset_nextents_get(eset_t *eset, pszind_t ind);
/* Get the sum total bytes of the extents in the given page size index. */
size_t eset_nbytes_get(eset_t *eset, pszind_t ind);
-void eset_insert_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata);
-void eset_remove_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata);
+void eset_insert(eset_t *eset, edata_t *edata);
+void eset_remove(eset_t *eset, edata_t *edata);
/*
* Select an extent from this eset of the given size and alignment. Returns
* null if no such item could be found.
*/
-edata_t *eset_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t esize,
- size_t alignment);
-
-void eset_prefork(tsdn_t *tsdn, eset_t *eset);
-void eset_postfork_parent(tsdn_t *tsdn, eset_t *eset);
-void eset_postfork_child(tsdn_t *tsdn, eset_t *eset);
+edata_t *eset_fit(eset_t *eset, size_t esize, size_t alignment);
#endif /* JEMALLOC_INTERNAL_ESET_H */
diff --git a/include/jemalloc/internal/extent2.h b/include/jemalloc/internal/extent2.h
index 08443366..80e789e4 100644
--- a/include/jemalloc/internal/extent2.h
+++ b/include/jemalloc/internal/extent2.h
@@ -1,8 +1,8 @@
#ifndef JEMALLOC_INTERNAL_EXTENT2_H
#define JEMALLOC_INTERNAL_EXTENT2_H
+#include "jemalloc/internal/ecache.h"
#include "jemalloc/internal/ehooks.h"
-#include "jemalloc/internal/eset.h"
#include "jemalloc/internal/ph.h"
#include "jemalloc/internal/rtree.h"
@@ -27,12 +27,12 @@ extern size_t opt_lg_extent_max_active_fit;
extern rtree_t extents_rtree;
edata_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment,
+ ecache_t *ecache, void *new_addr, size_t size, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit);
void extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- eset_t *eset, edata_t *edata);
+ ecache_t *ecache, edata_t *edata);
edata_t *extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- eset_t *eset, size_t npages_min);
+ ecache_t *ecache, size_t npages_min);
edata_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit);
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index 23312d3b..7b2e84a9 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -44,6 +44,7 @@
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index 76c16c5a..338962b3 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -44,6 +44,7 @@
+
diff --git a/src/arena.c b/src/arena.c
index a23419a3..2652207a 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -56,7 +56,7 @@ static unsigned huge_arena_ind;
*/
static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena,
- arena_decay_t *decay, eset_t *eset, bool all, size_t npages_limit,
+ arena_decay_t *decay, ecache_t *ecache, bool all, size_t npages_limit,
size_t npages_decay_max, bool is_background_thread);
static bool arena_decay_dirty(tsdn_t *tsdn, arena_t *arena,
bool is_background_thread, bool all);
@@ -76,8 +76,8 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
*dirty_decay_ms = arena_dirty_decay_ms_get(arena);
*muzzy_decay_ms = arena_muzzy_decay_ms_get(arena);
*nactive += atomic_load_zu(&arena->nactive, ATOMIC_RELAXED);
- *ndirty += eset_npages_get(&arena->eset_dirty);
- *nmuzzy += eset_npages_get(&arena->eset_muzzy);
+ *ndirty += ecache_npages_get(&arena->ecache_dirty);
+ *nmuzzy += ecache_npages_get(&arena->ecache_muzzy);
}
void
@@ -100,7 +100,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_stats_accum_zu(&astats->mapped, base_mapped
+ arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped));
arena_stats_accum_zu(&astats->retained,
- eset_npages_get(&arena->eset_retained) << LG_PAGE);
+ ecache_npages_get(&arena->ecache_retained) << LG_PAGE);
atomic_store_zu(&astats->edata_avail,
atomic_load_zu(&arena->edata_cache.count, ATOMIC_RELAXED),
@@ -131,8 +131,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_stats_accum_zu(&astats->metadata_thp, metadata_thp);
arena_stats_accum_zu(&astats->resident, base_resident +
(((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) +
- eset_npages_get(&arena->eset_dirty) +
- eset_npages_get(&arena->eset_muzzy)) << LG_PAGE)));
+ ecache_npages_get(&arena->ecache_dirty) +
+ ecache_npages_get(&arena->ecache_muzzy)) << LG_PAGE)));
arena_stats_accum_zu(&astats->abandoned_vm, atomic_load_zu(
&arena->stats.abandoned_vm, ATOMIC_RELAXED));
@@ -174,12 +174,12 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
for (pszind_t i = 0; i < SC_NPSIZES; i++) {
size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
retained_bytes;
- dirty = eset_nextents_get(&arena->eset_dirty, i);
- muzzy = eset_nextents_get(&arena->eset_muzzy, i);
- retained = eset_nextents_get(&arena->eset_retained, i);
- dirty_bytes = eset_nbytes_get(&arena->eset_dirty, i);
- muzzy_bytes = eset_nbytes_get(&arena->eset_muzzy, i);
- retained_bytes = eset_nbytes_get(&arena->eset_retained, i);
+ dirty = ecache_nextents_get(&arena->ecache_dirty, i);
+ muzzy = ecache_nextents_get(&arena->ecache_muzzy, i);
+ retained = ecache_nextents_get(&arena->ecache_retained, i);
+ dirty_bytes = ecache_nbytes_get(&arena->ecache_dirty, i);
+ muzzy_bytes = ecache_nbytes_get(&arena->ecache_muzzy, i);
+ retained_bytes = ecache_nbytes_get(&arena->ecache_retained, i);
atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED);
atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED);
@@ -226,11 +226,11 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
READ_ARENA_MUTEX_PROF_DATA(edata_cache.mtx,
arena_prof_mutex_extent_avail)
- READ_ARENA_MUTEX_PROF_DATA(eset_dirty.mtx,
+ READ_ARENA_MUTEX_PROF_DATA(ecache_dirty.mtx,
arena_prof_mutex_extents_dirty)
- READ_ARENA_MUTEX_PROF_DATA(eset_muzzy.mtx,
+ READ_ARENA_MUTEX_PROF_DATA(ecache_muzzy.mtx,
arena_prof_mutex_extents_muzzy)
- READ_ARENA_MUTEX_PROF_DATA(eset_retained.mtx,
+ READ_ARENA_MUTEX_PROF_DATA(ecache_retained.mtx,
arena_prof_mutex_extents_retained)
READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx,
arena_prof_mutex_decay_dirty)
@@ -258,7 +258,7 @@ arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
- extents_dalloc(tsdn, arena, ehooks, &arena->eset_dirty, edata);
+ extents_dalloc(tsdn, arena, ehooks, &arena->ecache_dirty, edata);
if (arena_dirty_decay_ms_get(arena) == 0) {
arena_decay_dirty(tsdn, arena, false, true);
} else {
@@ -434,10 +434,11 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
szind_t szind = sz_size2index(usize);
size_t mapped_add;
bool commit = true;
- edata_t *edata = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
- NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit);
+ edata_t *edata = extents_alloc(tsdn, arena, ehooks,
+ &arena->ecache_dirty, NULL, usize, sz_large_pad, alignment, false,
+ szind, zero, &commit);
if (edata == NULL && arena_may_have_muzzy(arena)) {
- edata = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
+ edata = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy,
NULL, usize, sz_large_pad, alignment, false, szind, zero,
&commit);
}
@@ -606,10 +607,10 @@ arena_decay_backlog_update(arena_decay_t *decay, uint64_t nadvance_u64,
static void
arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
- eset_t *eset, size_t current_npages, size_t npages_limit,
+ ecache_t *ecache, size_t current_npages, size_t npages_limit,
bool is_background_thread) {
if (current_npages > npages_limit) {
- arena_decay_to_limit(tsdn, arena, decay, eset, false,
+ arena_decay_to_limit(tsdn, arena, decay, ecache, false,
npages_limit, current_npages - npages_limit,
is_background_thread);
}
@@ -641,8 +642,8 @@ arena_decay_epoch_advance_helper(arena_decay_t *decay, const nstime_t *time,
static void
arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
- eset_t *eset, const nstime_t *time, bool is_background_thread) {
- size_t current_npages = eset_npages_get(eset);
+ ecache_t *ecache, const nstime_t *time, bool is_background_thread) {
+ size_t current_npages = ecache_npages_get(ecache);
arena_decay_epoch_advance_helper(decay, time, current_npages);
size_t npages_limit = arena_decay_backlog_npages_limit(decay);
@@ -651,7 +652,7 @@ arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
current_npages;
if (!background_thread_enabled() || is_background_thread) {
- arena_decay_try_purge(tsdn, arena, decay, eset,
+ arena_decay_try_purge(tsdn, arena, decay, ecache,
current_npages, npages_limit, is_background_thread);
}
}
@@ -708,15 +709,15 @@ arena_decay_ms_valid(ssize_t decay_ms) {
static bool
arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
- eset_t *eset, bool is_background_thread) {
+ ecache_t *ecache, bool is_background_thread) {
malloc_mutex_assert_owner(tsdn, &decay->mtx);
/* Purge all or nothing if the option is disabled. */
ssize_t decay_ms = arena_decay_ms_read(decay);
if (decay_ms <= 0) {
if (decay_ms == 0) {
- arena_decay_to_limit(tsdn, arena, decay, eset, false,
- 0, eset_npages_get(eset),
+ arena_decay_to_limit(tsdn, arena, decay, ecache, false,
+ 0, ecache_npages_get(ecache),
is_background_thread);
}
return false;
@@ -751,11 +752,11 @@ arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
*/
bool advance_epoch = arena_decay_deadline_reached(decay, &time);
if (advance_epoch) {
- arena_decay_epoch_advance(tsdn, arena, decay, eset, &time,
+ arena_decay_epoch_advance(tsdn, arena, decay, ecache, &time,
is_background_thread);
} else if (is_background_thread) {
- arena_decay_try_purge(tsdn, arena, decay, eset,
- eset_npages_get(eset),
+ arena_decay_try_purge(tsdn, arena, decay, ecache,
+ ecache_npages_get(ecache),
arena_decay_backlog_npages_limit(decay),
is_background_thread);
}
@@ -780,7 +781,7 @@ arena_muzzy_decay_ms_get(arena_t *arena) {
static bool
arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
- eset_t *eset, ssize_t decay_ms) {
+ ecache_t *ecache, ssize_t decay_ms) {
if (!arena_decay_ms_valid(decay_ms)) {
return true;
}
@@ -795,7 +796,7 @@ arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
* arbitrary change during initial arena configuration.
*/
arena_decay_reinit(decay, decay_ms);
- arena_maybe_decay(tsdn, arena, decay, eset, false);
+ arena_maybe_decay(tsdn, arena, decay, ecache, false);
malloc_mutex_unlock(tsdn, &decay->mtx);
return false;
@@ -805,19 +806,19 @@ bool
arena_dirty_decay_ms_set(tsdn_t *tsdn, arena_t *arena,
ssize_t decay_ms) {
return arena_decay_ms_set(tsdn, arena, &arena->decay_dirty,
- &arena->eset_dirty, decay_ms);
+ &arena->ecache_dirty, decay_ms);
}
bool
arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena,
ssize_t decay_ms) {
return arena_decay_ms_set(tsdn, arena, &arena->decay_muzzy,
- &arena->eset_muzzy, decay_ms);
+ &arena->ecache_muzzy, decay_ms);
}
static size_t
arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
- ehooks_t *ehooks, eset_t *eset, size_t npages_limit,
+ ehooks_t *ehooks, ecache_t *ecache, size_t npages_limit,
size_t npages_decay_max, edata_list_t *decay_extents) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
@@ -826,7 +827,7 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
size_t nstashed = 0;
edata_t *edata;
while (nstashed < npages_decay_max &&
- (edata = extents_evict(tsdn, arena, ehooks, eset, npages_limit))
+ (edata = extents_evict(tsdn, arena, ehooks, ecache, npages_limit))
!= NULL) {
edata_list_append(decay_extents, edata);
nstashed += edata_size_get(edata) >> LG_PAGE;
@@ -836,8 +837,8 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
static size_t
arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- arena_decay_t *decay, eset_t *eset, bool all, edata_list_t *decay_extents,
- bool is_background_thread) {
+ arena_decay_t *decay, ecache_t *ecache, bool all,
+ edata_list_t *decay_extents, bool is_background_thread) {
size_t nmadvise, nunmapped;
size_t npurged;
@@ -856,7 +857,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
size_t npages = edata_size_get(edata) >> LG_PAGE;
npurged += npages;
edata_list_remove(decay_extents, edata);
- switch (eset_state_get(eset)) {
+ switch (eset_state_get(&ecache->eset)) {
case extent_state_active:
not_reached();
case extent_state_dirty:
@@ -864,7 +865,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
!extent_purge_lazy_wrapper(tsdn, arena,
ehooks, edata, 0, edata_size_get(edata))) {
extents_dalloc(tsdn, arena, ehooks,
- &arena->eset_muzzy, edata);
+ &arena->ecache_muzzy, edata);
arena_background_thread_inactivity_check(tsdn,
arena, is_background_thread);
break;
@@ -900,14 +901,14 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
/*
* npages_limit: Decay at most npages_decay_max pages without violating the
- * invariant: (eset_npages_get(extents) >= npages_limit). We need an upper
+ * invariant: (ecache_npages_get(ecache) >= npages_limit). We need an upper
* bound on number of pages in order to prevent unbounded growth (namely in
* stashed), otherwise unbounded new pages could be added to extents during the
* current decay run, so that the purging thread never finishes.
*/
static void
arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
- eset_t *eset, bool all, size_t npages_limit, size_t npages_decay_max,
+ ecache_t *ecache, bool all, size_t npages_limit, size_t npages_decay_max,
bool is_background_thread) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 1);
@@ -924,11 +925,11 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
edata_list_t decay_extents;
edata_list_init(&decay_extents);
- size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, eset,
+ size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, ecache,
npages_limit, npages_decay_max, &decay_extents);
if (npurge != 0) {
size_t npurged = arena_decay_stashed(tsdn, arena, ehooks, decay,
- eset, all, &decay_extents, is_background_thread);
+ ecache, all, &decay_extents, is_background_thread);
assert(npurged == npurge);
}
@@ -938,11 +939,11 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
static bool
arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
- eset_t *eset, bool is_background_thread, bool all) {
+ ecache_t *ecache, bool is_background_thread, bool all) {
if (all) {
malloc_mutex_lock(tsdn, &decay->mtx);
- arena_decay_to_limit(tsdn, arena, decay, eset, all, 0,
- eset_npages_get(eset), is_background_thread);
+ arena_decay_to_limit(tsdn, arena, decay, ecache, all, 0,
+ ecache_npages_get(ecache), is_background_thread);
malloc_mutex_unlock(tsdn, &decay->mtx);
return false;
@@ -953,7 +954,7 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
return true;
}
- bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, eset,
+ bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, ecache,
is_background_thread);
size_t npages_new;
if (epoch_advanced) {
@@ -975,18 +976,18 @@ static bool
arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
bool all) {
return arena_decay_impl(tsdn, arena, &arena->decay_dirty,
- &arena->eset_dirty, is_background_thread, all);
+ &arena->ecache_dirty, is_background_thread, all);
}
static bool
arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
bool all) {
- if (eset_npages_get(&arena->eset_muzzy) == 0 &&
+ if (ecache_npages_get(&arena->ecache_muzzy) == 0 &&
arena_muzzy_decay_ms_get(arena) <= 0) {
return false;
}
return arena_decay_impl(tsdn, arena, &arena->decay_muzzy,
- &arena->eset_muzzy, is_background_thread, all);
+ &arena->ecache_muzzy, is_background_thread, all);
}
void
@@ -1157,7 +1158,7 @@ arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) {
ehooks_t *ehooks = arena_get_ehooks(arena);
edata_t *edata;
while ((edata = extents_evict(tsdn, arena, ehooks,
- &arena->eset_retained, 0)) != NULL) {
+ &arena->ecache_retained, 0)) != NULL) {
extent_destroy_wrapper(tsdn, arena, ehooks, edata);
}
}
@@ -1173,8 +1174,8 @@ arena_destroy(tsd_t *tsd, arena_t *arena) {
* Furthermore, the caller (arena_i_destroy_ctl()) purged all cached
* extents, so only retained extents may remain.
*/
- assert(eset_npages_get(&arena->eset_dirty) == 0);
- assert(eset_npages_get(&arena->eset_muzzy) == 0);
+ assert(ecache_npages_get(&arena->ecache_dirty) == 0);
+ assert(ecache_npages_get(&arena->ecache_muzzy) == 0);
/* Deallocate retained memory. */
arena_destroy_retained(tsd_tsdn(tsd), arena);
@@ -1230,10 +1231,10 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard
szind_t szind = sz_size2index(bin_info->reg_size);
bool zero = false;
bool commit = true;
- edata_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
+ edata_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->ecache_dirty,
NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit);
if (slab == NULL && arena_may_have_muzzy(arena)) {
- slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
+ slab = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy,
NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero,
&commit);
}
@@ -1917,14 +1918,14 @@ arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit,
}
}
- malloc_mutex_lock(tsd_tsdn(tsd), &arena->extent_grow_mtx);
+ malloc_mutex_lock(tsd_tsdn(tsd), &arena->ecache_grow.mtx);
if (old_limit != NULL) {
- *old_limit = sz_pind2sz(arena->retain_grow_limit);
+ *old_limit = sz_pind2sz(arena->ecache_grow.limit);
}
if (new_limit != NULL) {
- arena->retain_grow_limit = new_ind;
+ arena->ecache_grow.limit = new_ind;
}
- malloc_mutex_unlock(tsd_tsdn(tsd), &arena->extent_grow_mtx);
+ malloc_mutex_unlock(tsd_tsdn(tsd), &arena->ecache_grow.mtx);
return false;
}
@@ -2016,14 +2017,14 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
* are likely to be reused soon after deallocation, and the cost of
* merging/splitting extents is non-trivial.
*/
- if (eset_init(tsdn, &arena->eset_dirty, extent_state_dirty, true)) {
+ if (ecache_init(tsdn, &arena->ecache_dirty, extent_state_dirty, true)) {
goto label_error;
}
/*
* Coalesce muzzy extents immediately, because operations on them are in
* the critical path much less often than for dirty extents.
*/
- if (eset_init(tsdn, &arena->eset_muzzy, extent_state_muzzy, false)) {
+ if (ecache_init(tsdn, &arena->ecache_muzzy, extent_state_muzzy, false)) {
goto label_error;
}
/*
@@ -2032,7 +2033,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
* coalescing), but also because operations on retained extents are not
* in the critical path.
*/
- if (eset_init(tsdn, &arena->eset_retained, extent_state_retained,
+ if (ecache_init(tsdn, &arena->ecache_retained, extent_state_retained,
false)) {
goto label_error;
}
@@ -2046,10 +2047,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
goto label_error;
}
- arena->extent_grow_next = sz_psz2ind(HUGEPAGE);
- arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS);
- if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow",
- WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
+ if (ecache_grow_init(tsdn, &arena->ecache_grow)) {
goto label_error;
}
@@ -2187,14 +2185,14 @@ arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
void
arena_prefork2(tsdn_t *tsdn, arena_t *arena) {
- malloc_mutex_prefork(tsdn, &arena->extent_grow_mtx);
+ ecache_grow_prefork(tsdn, &arena->ecache_grow);
}
void
arena_prefork3(tsdn_t *tsdn, arena_t *arena) {
- eset_prefork(tsdn, &arena->eset_dirty);
- eset_prefork(tsdn, &arena->eset_muzzy);
- eset_prefork(tsdn, &arena->eset_retained);
+ ecache_prefork(tsdn, &arena->ecache_dirty);
+ ecache_prefork(tsdn, &arena->ecache_muzzy);
+ ecache_prefork(tsdn, &arena->ecache_retained);
}
void
@@ -2234,10 +2232,10 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
base_postfork_parent(tsdn, arena->base);
edata_cache_postfork_parent(tsdn, &arena->edata_cache);
- eset_postfork_parent(tsdn, &arena->eset_dirty);
- eset_postfork_parent(tsdn, &arena->eset_muzzy);
- eset_postfork_parent(tsdn, &arena->eset_retained);
- malloc_mutex_postfork_parent(tsdn, &arena->extent_grow_mtx);
+ ecache_postfork_parent(tsdn, &arena->ecache_dirty);
+ ecache_postfork_parent(tsdn, &arena->ecache_muzzy);
+ ecache_postfork_parent(tsdn, &arena->ecache_retained);
+ ecache_grow_postfork_parent(tsdn, &arena->ecache_grow);
malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx);
if (config_stats) {
@@ -2280,10 +2278,10 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
base_postfork_child(tsdn, arena->base);
edata_cache_postfork_child(tsdn, &arena->edata_cache);
- eset_postfork_child(tsdn, &arena->eset_dirty);
- eset_postfork_child(tsdn, &arena->eset_muzzy);
- eset_postfork_child(tsdn, &arena->eset_retained);
- malloc_mutex_postfork_child(tsdn, &arena->extent_grow_mtx);
+ ecache_postfork_child(tsdn, &arena->ecache_dirty);
+ ecache_postfork_child(tsdn, &arena->ecache_muzzy);
+ ecache_postfork_child(tsdn, &arena->ecache_retained);
+ ecache_grow_postfork_child(tsdn, &arena->ecache_grow);
malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx);
if (config_stats) {
diff --git a/src/background_thread.c b/src/background_thread.c
index 400dae53..90b027ed 100644
--- a/src/background_thread.c
+++ b/src/background_thread.c
@@ -114,7 +114,7 @@ decay_npurge_after_interval(arena_decay_t *decay, size_t interval) {
static uint64_t
arena_decay_compute_purge_interval_impl(tsdn_t *tsdn, arena_decay_t *decay,
- eset_t *eset) {
+ ecache_t *ecache) {
if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
/* Use minimal interval if decay is contended. */
return BACKGROUND_THREAD_MIN_INTERVAL_NS;
@@ -130,7 +130,7 @@ arena_decay_compute_purge_interval_impl(tsdn_t *tsdn, arena_decay_t *decay,
uint64_t decay_interval_ns = nstime_ns(&decay->interval);
assert(decay_interval_ns > 0);
- size_t npages = eset_npages_get(eset);
+ size_t npages = ecache_npages_get(ecache);
if (npages == 0) {
unsigned i;
for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) {
@@ -202,12 +202,12 @@ static uint64_t
arena_decay_compute_purge_interval(tsdn_t *tsdn, arena_t *arena) {
uint64_t i1, i2;
i1 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_dirty,
- &arena->eset_dirty);
+ &arena->ecache_dirty);
if (i1 == BACKGROUND_THREAD_MIN_INTERVAL_NS) {
return i1;
}
i2 = arena_decay_compute_purge_interval_impl(tsdn, &arena->decay_muzzy,
- &arena->eset_muzzy);
+ &arena->ecache_muzzy);
return i1 < i2 ? i1 : i2;
}
@@ -717,8 +717,8 @@ background_thread_interval_check(tsdn_t *tsdn, arena_t *arena,
if (info->npages_to_purge_new > BACKGROUND_THREAD_NPAGES_THRESHOLD) {
should_signal = true;
} else if (unlikely(background_thread_indefinite_sleep(info)) &&
- (eset_npages_get(&arena->eset_dirty) > 0 ||
- eset_npages_get(&arena->eset_muzzy) > 0 ||
+ (ecache_npages_get(&arena->ecache_dirty) > 0 ||
+ ecache_npages_get(&arena->ecache_muzzy) > 0 ||
info->npages_to_purge_new > 0)) {
should_signal = true;
} else {
diff --git a/src/ctl.c b/src/ctl.c
index a58b22ba..eee12770 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -3011,9 +3011,9 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
}
MUTEX_PROF_RESET(arena->large_mtx);
MUTEX_PROF_RESET(arena->edata_cache.mtx);
- MUTEX_PROF_RESET(arena->eset_dirty.mtx);
- MUTEX_PROF_RESET(arena->eset_muzzy.mtx);
- MUTEX_PROF_RESET(arena->eset_retained.mtx);
+ MUTEX_PROF_RESET(arena->ecache_dirty.mtx);
+ MUTEX_PROF_RESET(arena->ecache_muzzy.mtx);
+ MUTEX_PROF_RESET(arena->ecache_retained.mtx);
MUTEX_PROF_RESET(arena->decay_dirty.mtx);
MUTEX_PROF_RESET(arena->decay_muzzy.mtx);
MUTEX_PROF_RESET(arena->tcache_ql_mtx);
diff --git a/src/ecache.c b/src/ecache.c
new file mode 100644
index 00000000..7155de35
--- /dev/null
+++ b/src/ecache.c
@@ -0,0 +1,54 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+bool
+ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state,
+ bool delay_coalesce) {
+ if (malloc_mutex_init(&ecache->mtx, "extents", WITNESS_RANK_EXTENTS,
+ malloc_mutex_rank_exclusive)) {
+ return true;
+ }
+ eset_init(&ecache->eset, state, delay_coalesce);
+ return false;
+}
+
+void
+ecache_prefork(tsdn_t *tsdn, ecache_t *ecache) {
+ malloc_mutex_prefork(tsdn, &ecache->mtx);
+}
+
+void
+ecache_postfork_parent(tsdn_t *tsdn, ecache_t *ecache) {
+ malloc_mutex_postfork_parent(tsdn, &ecache->mtx);
+}
+
+void
+ecache_postfork_child(tsdn_t *tsdn, ecache_t *ecache) {
+ malloc_mutex_postfork_child(tsdn, &ecache->mtx);
+}
+
+bool
+ecache_grow_init(tsdn_t *tsdn, ecache_grow_t *ecache_grow) {
+ ecache_grow->next = sz_psz2ind(HUGEPAGE);
+ ecache_grow->limit = sz_psz2ind(SC_LARGE_MAXCLASS);
+ if (malloc_mutex_init(&ecache_grow->mtx, "extent_grow",
+ WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
+ return true;
+ }
+ return false;
+}
+
+void
+ecache_grow_prefork(tsdn_t *tsdn, ecache_grow_t *ecache_grow) {
+ malloc_mutex_prefork(tsdn, &ecache_grow->mtx);
+}
+
+void
+ecache_grow_postfork_parent(tsdn_t *tsdn, ecache_grow_t *ecache_grow) {
+ malloc_mutex_postfork_parent(tsdn, &ecache_grow->mtx);
+}
+
+void
+ecache_grow_postfork_child(tsdn_t *tsdn, ecache_grow_t *ecache_grow) {
+ malloc_mutex_postfork_child(tsdn, &ecache_grow->mtx);
+}
diff --git a/src/eset.c b/src/eset.c
index 88b9c8c7..e71868c8 100644
--- a/src/eset.c
+++ b/src/eset.c
@@ -8,13 +8,9 @@
const bitmap_info_t eset_bitmap_info =
BITMAP_INFO_INITIALIZER(SC_NPSIZES+1);
-bool
-eset_init(tsdn_t *tsdn, eset_t *eset, extent_state_t state,
+void
+eset_init(eset_t *eset, extent_state_t state,
bool delay_coalesce) {
- if (malloc_mutex_init(&eset->mtx, "extents", WITNESS_RANK_EXTENTS,
- malloc_mutex_rank_exclusive)) {
- return true;
- }
for (unsigned i = 0; i < SC_NPSIZES + 1; i++) {
edata_heap_new(&eset->heaps[i]);
}
@@ -23,7 +19,6 @@ eset_init(tsdn_t *tsdn, eset_t *eset, extent_state_t state,
atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED);
eset->state = state;
eset->delay_coalesce = delay_coalesce;
- return false;
}
extent_state_t
@@ -63,8 +58,7 @@ eset_stats_sub(eset_t *eset, pszind_t pind, size_t sz) {
}
void
-eset_insert_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) {
- malloc_mutex_assert_owner(tsdn, &eset->mtx);
+eset_insert(eset_t *eset, edata_t *edata) {
assert(edata_state_get(edata) == eset->state);
size_t size = edata_size_get(edata);
@@ -94,8 +88,7 @@ eset_insert_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) {
}
void
-eset_remove_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) {
- malloc_mutex_assert_owner(tsdn, &eset->mtx);
+eset_remove(eset_t *eset, edata_t *edata) {
assert(edata_state_get(edata) == eset->state);
size_t size = edata_size_get(edata);
@@ -114,9 +107,13 @@ eset_remove_locked(tsdn_t *tsdn, eset_t *eset, edata_t *edata) {
edata_list_remove(&eset->lru, edata);
size_t npages = size >> LG_PAGE;
/*
- * As in eset_insert_locked, we hold eset->mtx and so don't need atomic
+ * As in eset_insert, we hold eset->mtx and so don't need atomic
* operations for updating eset->npages.
*/
+ /*
+ * This class is not thread-safe in general; we rely on external
+ * synchronization for all mutating operations.
+ */
size_t cur_extents_npages =
atomic_load_zu(&eset->npages, ATOMIC_RELAXED);
assert(cur_extents_npages >= npages);
@@ -166,7 +163,7 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size,
* large enough.
*/
static edata_t *
-eset_first_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t size) {
+eset_first_fit(eset_t *eset, size_t size) {
edata_t *ret = NULL;
pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(size));
@@ -211,16 +208,14 @@ eset_first_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t size) {
}
edata_t *
-eset_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t esize, size_t alignment) {
- malloc_mutex_assert_owner(tsdn, &eset->mtx);
-
+eset_fit(eset_t *eset, size_t esize, size_t alignment) {
size_t max_size = esize + PAGE_CEILING(alignment) - PAGE;
/* Beware size_t wrap-around. */
if (max_size < esize) {
return NULL;
}
- edata_t *edata = eset_first_fit_locked(tsdn, eset, max_size);
+ edata_t *edata = eset_first_fit(eset, max_size);
if (alignment > PAGE && edata == NULL) {
/*
@@ -233,18 +228,3 @@ eset_fit_locked(tsdn_t *tsdn, eset_t *eset, size_t esize, size_t alignment) {
return edata;
}
-
-void
-eset_prefork(tsdn_t *tsdn, eset_t *eset) {
- malloc_mutex_prefork(tsdn, &eset->mtx);
-}
-
-void
-eset_postfork_parent(tsdn_t *tsdn, eset_t *eset) {
- malloc_mutex_postfork_parent(tsdn, &eset->mtx);
-}
-
-void
-eset_postfork_child(tsdn_t *tsdn, eset_t *eset) {
- malloc_mutex_postfork_child(tsdn, &eset->mtx);
-}
diff --git a/src/extent2.c b/src/extent2.c
index 21f9cdbd..3b47e070 100644
--- a/src/extent2.c
+++ b/src/extent2.c
@@ -45,13 +45,13 @@ static atomic_zu_t highpages;
static void extent_deregister(tsdn_t *tsdn, edata_t *edata);
static edata_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- eset_t *eset, void *new_addr, size_t usize, size_t pad, size_t alignment,
+ ecache_t *ecache, void *new_addr, size_t usize, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained);
static edata_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
- ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata,
+ ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata,
bool *coalesced, bool growing_retained);
static void extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- eset_t *eset, edata_t *edata, bool growing_retained);
+ ecache_t *ecache, edata_t *edata, bool growing_retained);
/******************************************************************************/
@@ -165,22 +165,22 @@ extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
static bool
extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata) {
+ rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata) {
edata_state_set(edata, extent_state_active);
bool coalesced;
- edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, eset,
+ edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, ecache,
edata, &coalesced, false);
- edata_state_set(edata, eset_state_get(eset));
+ edata_state_set(edata, eset_state_get(&ecache->eset));
if (!coalesced) {
return true;
}
- eset_insert_locked(tsdn, eset, edata);
+ eset_insert(&ecache->eset, edata);
return false;
}
edata_t *
-extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit) {
assert(size + pad != 0);
@@ -188,14 +188,14 @@ extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
- edata_t *edata = extent_recycle(tsdn, arena, ehooks, eset, new_addr,
+ edata_t *edata = extent_recycle(tsdn, arena, ehooks, ecache, new_addr,
size, pad, alignment, slab, szind, zero, commit, false);
assert(edata == NULL || edata_dumpable_get(edata));
return edata;
}
void
-extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
edata_t *edata) {
assert(edata_base_get(edata) != NULL);
assert(edata_size_get(edata) != 0);
@@ -206,16 +206,16 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
edata_addr_set(edata, edata_base_get(edata));
edata_zeroed_set(edata, false);
- extent_record(tsdn, arena, ehooks, eset, edata, false);
+ extent_record(tsdn, arena, ehooks, ecache, edata, false);
}
edata_t *
-extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
size_t npages_min) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
- malloc_mutex_lock(tsdn, &eset->mtx);
+ malloc_mutex_lock(tsdn, &ecache->mtx);
/*
* Get the LRU coalesced extent, if any. If coalescing was delayed,
@@ -224,24 +224,23 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
edata_t *edata;
while (true) {
/* Get the LRU extent, if any. */
- edata = edata_list_first(&eset->lru);
+ edata = edata_list_first(&ecache->eset.lru);
if (edata == NULL) {
goto label_return;
}
/* Check the eviction limit. */
- size_t extents_npages = atomic_load_zu(&eset->npages,
- ATOMIC_RELAXED);
+ size_t extents_npages = ecache_npages_get(ecache);
if (extents_npages <= npages_min) {
edata = NULL;
goto label_return;
}
- eset_remove_locked(tsdn, eset, edata);
- if (!eset->delay_coalesce) {
+ eset_remove(&ecache->eset, edata);
+ if (!ecache->eset.delay_coalesce) {
break;
}
/* Try to coalesce. */
if (extent_try_delayed_coalesce(tsdn, arena, ehooks, rtree_ctx,
- eset, edata)) {
+ ecache, edata)) {
break;
}
/*
@@ -254,7 +253,7 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
* Either mark the extent active or deregister it to protect against
* concurrent operations.
*/
- switch (eset_state_get(eset)) {
+ switch (eset_state_get(&ecache->eset)) {
case extent_state_active:
not_reached();
case extent_state_dirty:
@@ -269,7 +268,7 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
}
label_return:
- malloc_mutex_unlock(tsdn, &eset->mtx);
+ malloc_mutex_unlock(tsdn, &ecache->mtx);
return edata;
}
@@ -278,8 +277,8 @@ label_return:
* indicates OOM), e.g. when trying to split an existing extent.
*/
static void
-extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- edata_t *edata, bool growing_retained) {
+extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ ecache_t *ecache, edata_t *edata, bool growing_retained) {
size_t sz = edata_size_get(edata);
if (config_stats) {
arena_stats_accum_zu(&arena->stats.abandoned_vm, sz);
@@ -288,7 +287,7 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
* Leak extent after making sure its pages have already been purged, so
* that this is only a virtual memory leak.
*/
- if (eset_state_get(eset) == extent_state_dirty) {
+ if (eset_state_get(&ecache->eset) == extent_state_dirty) {
if (extent_purge_lazy_impl(tsdn, arena, ehooks, edata, 0, sz,
growing_retained)) {
extent_purge_forced_impl(tsdn, arena, ehooks, edata, 0,
@@ -299,30 +298,30 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
}
static void
-extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
+extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache,
edata_t *edata) {
assert(edata_arena_ind_get(edata) == arena_ind_get(arena));
assert(edata_state_get(edata) == extent_state_active);
- edata_state_set(edata, eset_state_get(eset));
- eset_insert_locked(tsdn, eset, edata);
+ edata_state_set(edata, eset_state_get(&ecache->eset));
+ eset_insert(&ecache->eset, edata);
}
static void
-extent_deactivate(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
+extent_deactivate(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache,
edata_t *edata) {
- malloc_mutex_lock(tsdn, &eset->mtx);
- extent_deactivate_locked(tsdn, arena, eset, edata);
- malloc_mutex_unlock(tsdn, &eset->mtx);
+ malloc_mutex_lock(tsdn, &ecache->mtx);
+ extent_deactivate_locked(tsdn, arena, ecache, edata);
+ malloc_mutex_unlock(tsdn, &ecache->mtx);
}
static void
-extent_activate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
+extent_activate_locked(tsdn_t *tsdn, arena_t *arena, ecache_t *ecache,
edata_t *edata) {
assert(edata_arena_ind_get(edata) == arena_ind_get(arena));
- assert(edata_state_get(edata) == eset_state_get(eset));
+ assert(edata_state_get(edata) == eset_state_get(&ecache->eset));
- eset_remove_locked(tsdn, eset, edata);
+ eset_remove(&ecache->eset, edata);
edata_state_set(edata, extent_state_active);
}
@@ -515,12 +514,12 @@ extent_deregister_no_gdump_sub(tsdn_t *tsdn, edata_t *edata) {
}
/*
- * Tries to find and remove an extent from eset that can be used for the
+ * Tries to find and remove an extent from ecache that can be used for the
* given allocation request.
*/
static edata_t *
extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
+ rtree_ctx_t *rtree_ctx, ecache_t *ecache, void *new_addr, size_t size,
size_t pad, size_t alignment, bool slab, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
@@ -543,7 +542,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
}
size_t esize = size + pad;
- malloc_mutex_lock(tsdn, &eset->mtx);
+ malloc_mutex_lock(tsdn, &ecache->mtx);
edata_t *edata;
if (new_addr != NULL) {
edata = extent_lock_edata_from_addr(tsdn, rtree_ctx, new_addr,
@@ -557,21 +556,22 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
assert(edata_base_get(edata) == new_addr);
if (edata_arena_ind_get(edata) != arena_ind_get(arena)
|| edata_size_get(edata) < esize
- || edata_state_get(edata) != eset_state_get(eset)) {
+ || edata_state_get(edata)
+ != eset_state_get(&ecache->eset)) {
edata = NULL;
}
extent_unlock_edata(tsdn, unlock_edata);
}
} else {
- edata = eset_fit_locked(tsdn, eset, esize, alignment);
+ edata = eset_fit(&ecache->eset, esize, alignment);
}
if (edata == NULL) {
- malloc_mutex_unlock(tsdn, &eset->mtx);
+ malloc_mutex_unlock(tsdn, &ecache->mtx);
return NULL;
}
- extent_activate_locked(tsdn, arena, eset, edata);
- malloc_mutex_unlock(tsdn, &eset->mtx);
+ extent_activate_locked(tsdn, arena, ecache, edata);
+ malloc_mutex_unlock(tsdn, &ecache->mtx);
return edata;
}
@@ -580,7 +580,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
* Given an allocation request and an extent guaranteed to be able to satisfy
* it, this splits off lead and trail extents, leaving edata pointing to an
* extent satisfying the allocation.
- * This function doesn't put lead or trail into any eset_t; it's the caller's
+ * This function doesn't put lead or trail into any ecache; it's the caller's
* job to ensure that they can be reused.
*/
typedef enum {
@@ -676,11 +676,11 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
* This fulfills the indicated allocation request out of the given extent (which
* the caller should have ensured was big enough). If there's any unused space
* before or after the resulting allocation, that space is given its own extent
- * and put back into eset.
+ * and put back into ecache.
*/
static edata_t *
extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
+ rtree_ctx_t *rtree_ctx, ecache_t *ecache, void *new_addr, size_t size,
size_t pad, size_t alignment, bool slab, szind_t szind, edata_t *edata,
bool growing_retained) {
edata_t *lead;
@@ -697,19 +697,19 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
&& !opt_retain) {
/*
* Split isn't supported (implies Windows w/o retain). Avoid
- * leaking the eset.
+ * leaking the extent.
*/
assert(to_leak != NULL && lead == NULL && trail == NULL);
- extent_deactivate(tsdn, arena, eset, to_leak);
+ extent_deactivate(tsdn, arena, ecache, to_leak);
return NULL;
}
if (result == extent_split_interior_ok) {
if (lead != NULL) {
- extent_deactivate(tsdn, arena, eset, lead);
+ extent_deactivate(tsdn, arena, ecache, lead);
}
if (trail != NULL) {
- extent_deactivate(tsdn, arena, eset, trail);
+ extent_deactivate(tsdn, arena, ecache, trail);
}
return edata;
} else {
@@ -724,7 +724,7 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
if (to_leak != NULL) {
void *leak = edata_base_get(to_leak);
extent_deregister_no_gdump_sub(tsdn, to_leak);
- extents_abandon_vm(tsdn, arena, ehooks, eset, to_leak,
+ extents_abandon_vm(tsdn, arena, ehooks, ecache, to_leak,
growing_retained);
assert(extent_lock_edata_from_addr(tsdn, rtree_ctx, leak,
false) == NULL);
@@ -736,10 +736,10 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
/*
* Tries to satisfy the given allocation request by reusing one of the extents
- * in the given eset_t.
+ * in the given ecache_t.
*/
static edata_t *
-extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
@@ -752,13 +752,13 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
edata_t *edata = extent_recycle_extract(tsdn, arena, ehooks,
- rtree_ctx, eset, new_addr, size, pad, alignment, slab,
+ rtree_ctx, ecache, new_addr, size, pad, alignment, slab,
growing_retained);
if (edata == NULL) {
return NULL;
}
- edata = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, eset,
+ edata = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, ecache,
new_addr, size, pad, alignment, slab, szind, edata,
growing_retained);
if (edata == NULL) {
@@ -768,7 +768,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
if (*commit && !edata_committed_get(edata)) {
if (extent_commit_impl(tsdn, arena, ehooks, edata, 0,
edata_size_get(edata), growing_retained)) {
- extent_record(tsdn, arena, ehooks, eset, edata,
+ extent_record(tsdn, arena, ehooks, ecache, edata,
growing_retained);
return NULL;
}
@@ -810,7 +810,7 @@ static edata_t *
extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
bool *zero, bool *commit) {
- malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx);
+ malloc_mutex_assert_owner(tsdn, &arena->ecache_grow.mtx);
assert(pad == 0 || !slab);
assert(!*zero || !slab);
@@ -825,15 +825,15 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
* satisfy this request.
*/
pszind_t egn_skip = 0;
- size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
+ size_t alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip);
while (alloc_size < alloc_size_min) {
egn_skip++;
- if (arena->extent_grow_next + egn_skip >=
+ if (arena->ecache_grow.next + egn_skip >=
sz_psz2ind(SC_LARGE_MAXCLASS)) {
/* Outside legal range. */
goto label_err;
}
- alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
+ alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip);
}
edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache,
@@ -881,11 +881,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
if (result == extent_split_interior_ok) {
if (lead != NULL) {
extent_record(tsdn, arena, ehooks,
- &arena->eset_retained, lead, true);
+ &arena->ecache_retained, lead, true);
}
if (trail != NULL) {
extent_record(tsdn, arena, ehooks,
- &arena->eset_retained, trail, true);
+ &arena->ecache_retained, trail, true);
}
} else {
/*
@@ -898,12 +898,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_gdump_add(tsdn, to_salvage);
}
extent_record(tsdn, arena, ehooks,
- &arena->eset_retained, to_salvage, true);
+ &arena->ecache_retained, to_salvage, true);
}
if (to_leak != NULL) {
extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, ehooks,
- &arena->eset_retained, to_leak, true);
+ &arena->ecache_retained, to_leak, true);
}
goto label_err;
}
@@ -912,7 +912,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
if (extent_commit_impl(tsdn, arena, ehooks, edata, 0,
edata_size_get(edata), true)) {
extent_record(tsdn, arena, ehooks,
- &arena->eset_retained, edata, true);
+ &arena->ecache_retained, edata, true);
goto label_err;
}
/* A successful commit should return zeroed memory. */
@@ -930,14 +930,14 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
* Increment extent_grow_next if doing so wouldn't exceed the allowed
* range.
*/
- if (arena->extent_grow_next + egn_skip + 1 <=
- arena->retain_grow_limit) {
- arena->extent_grow_next += egn_skip + 1;
+ if (arena->ecache_grow.next + egn_skip + 1 <=
+ arena->ecache_grow.limit) {
+ arena->ecache_grow.next += egn_skip + 1;
} else {
- arena->extent_grow_next = arena->retain_grow_limit;
+ arena->ecache_grow.next = arena->ecache_grow.limit;
}
/* All opportunities for failure are past. */
- malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+ malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx);
if (config_prof) {
/* Adjust gdump stats now that extent is final size. */
@@ -962,7 +962,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
return edata;
label_err:
- malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+ malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx);
return NULL;
}
@@ -973,13 +973,13 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
assert(size != 0);
assert(alignment != 0);
- malloc_mutex_lock(tsdn, &arena->extent_grow_mtx);
+ malloc_mutex_lock(tsdn, &arena->ecache_grow.mtx);
edata_t *edata = extent_recycle(tsdn, arena, ehooks,
- &arena->eset_retained, new_addr, size, pad, alignment, slab,
+ &arena->ecache_retained, new_addr, size, pad, alignment, slab,
szind, zero, commit, true);
if (edata != NULL) {
- malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+ malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx);
if (config_prof) {
extent_gdump_add(tsdn, edata);
}
@@ -988,9 +988,9 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
alignment, slab, szind, zero, commit);
/* extent_grow_retained() always releases extent_grow_mtx. */
} else {
- malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+ malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx);
}
- malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx);
+ malloc_mutex_assert_not_owner(tsdn, &arena->ecache_grow.mtx);
return edata;
}
@@ -1054,7 +1054,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
}
static bool
-extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner,
+extent_can_coalesce(arena_t *arena, ecache_t *ecache, const edata_t *inner,
const edata_t *outer) {
assert(edata_arena_ind_get(inner) == arena_ind_get(arena));
if (edata_arena_ind_get(outer) != arena_ind_get(arena)) {
@@ -1062,7 +1062,7 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner,
}
assert(edata_state_get(inner) == extent_state_active);
- if (edata_state_get(outer) != eset->state) {
+ if (edata_state_get(outer) != ecache->eset.state) {
return false;
}
@@ -1074,19 +1074,20 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const edata_t *inner,
}
static bool
-extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- edata_t *inner, edata_t *outer, bool forward, bool growing_retained) {
- assert(extent_can_coalesce(arena, eset, inner, outer));
+extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ ecache_t *ecache, edata_t *inner, edata_t *outer, bool forward,
+ bool growing_retained) {
+ assert(extent_can_coalesce(arena, ecache, inner, outer));
- extent_activate_locked(tsdn, arena, eset, outer);
+ extent_activate_locked(tsdn, arena, ecache, outer);
- malloc_mutex_unlock(tsdn, &eset->mtx);
+ malloc_mutex_unlock(tsdn, &ecache->mtx);
bool err = extent_merge_impl(tsdn, arena, ehooks,
forward ? inner : outer, forward ? outer : inner, growing_retained);
- malloc_mutex_lock(tsdn, &eset->mtx);
+ malloc_mutex_lock(tsdn, &ecache->mtx);
if (err) {
- extent_deactivate_locked(tsdn, arena, eset, outer);
+ extent_deactivate_locked(tsdn, arena, ecache, outer);
}
return err;
@@ -1094,7 +1095,7 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
static edata_t *
extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced,
+ rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced,
bool growing_retained, bool inactive_only) {
/*
* We avoid checking / locking inactive neighbors for large size
@@ -1114,19 +1115,19 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
edata_past_get(edata), inactive_only);
if (next != NULL) {
/*
- * eset->mtx only protects against races for
- * like-state eset, so call extent_can_coalesce()
+ * ecache->mtx only protects against races for
+ * like-state extents, so call extent_can_coalesce()
* before releasing next's pool lock.
*/
- bool can_coalesce = extent_can_coalesce(arena, eset,
+ bool can_coalesce = extent_can_coalesce(arena, ecache,
edata, next);
extent_unlock_edata(tsdn, next);
if (can_coalesce && !extent_coalesce(tsdn, arena,
- ehooks, eset, edata, next, true,
+ ehooks, ecache, edata, next, true,
growing_retained)) {
- if (eset->delay_coalesce) {
+ if (ecache->eset.delay_coalesce) {
/* Do minimal coalescing. */
*coalesced = true;
return edata;
@@ -1139,15 +1140,15 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
edata_t *prev = extent_lock_edata_from_addr(tsdn, rtree_ctx,
edata_before_get(edata), inactive_only);
if (prev != NULL) {
- bool can_coalesce = extent_can_coalesce(arena, eset,
+ bool can_coalesce = extent_can_coalesce(arena, ecache,
edata, prev);
extent_unlock_edata(tsdn, prev);
if (can_coalesce && !extent_coalesce(tsdn, arena,
- ehooks, eset, edata, prev, false,
+ ehooks, ecache, edata, prev, false,
growing_retained)) {
edata = prev;
- if (eset->delay_coalesce) {
+ if (ecache->eset.delay_coalesce) {
/* Do minimal coalescing. */
*coalesced = true;
return edata;
@@ -1157,7 +1158,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
}
} while (again);
- if (eset->delay_coalesce) {
+ if (ecache->eset.delay_coalesce) {
*coalesced = false;
}
return edata;
@@ -1165,35 +1166,35 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
static edata_t *
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced,
+ rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced,
bool growing_retained) {
- return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
+ return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, ecache,
edata, coalesced, growing_retained, false);
}
static edata_t *
extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata, bool *coalesced,
+ rtree_ctx_t *rtree_ctx, ecache_t *ecache, edata_t *edata, bool *coalesced,
bool growing_retained) {
- return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
+ return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, ecache,
edata, coalesced, growing_retained, true);
}
/*
* Does the metadata management portions of putting an unused extent into the
- * given eset_t (coalesces, deregisters slab interiors, the heap operations).
+ * given ecache_t (coalesces, deregisters slab interiors, the heap operations).
*/
static void
-extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
edata_t *edata, bool growing_retained) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
- assert((eset_state_get(eset) != extent_state_dirty &&
- eset_state_get(eset) != extent_state_muzzy) ||
+ assert((eset_state_get(&ecache->eset) != extent_state_dirty &&
+ eset_state_get(&ecache->eset) != extent_state_muzzy) ||
!edata_zeroed_get(edata));
- malloc_mutex_lock(tsdn, &eset->mtx);
+ malloc_mutex_lock(tsdn, &ecache->mtx);
edata_szind_set(edata, SC_NSIZES);
if (edata_slab_get(edata)) {
@@ -1204,29 +1205,29 @@ extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
assert(rtree_edata_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), true) == edata);
- if (!eset->delay_coalesce) {
+ if (!ecache->eset.delay_coalesce) {
edata = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx,
- eset, edata, NULL, growing_retained);
+ ecache, edata, NULL, growing_retained);
} else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) {
- assert(eset == &arena->eset_dirty);
- /* Always coalesce large eset eagerly. */
+ assert(ecache == &arena->ecache_dirty);
+ /* Always coalesce large extents eagerly. */
bool coalesced;
do {
assert(edata_state_get(edata) == extent_state_active);
edata = extent_try_coalesce_large(tsdn, arena, ehooks,
- rtree_ctx, eset, edata, &coalesced,
+ rtree_ctx, ecache, edata, &coalesced,
growing_retained);
} while (coalesced);
if (edata_size_get(edata) >= oversize_threshold) {
/* Shortcut to purge the oversize extent eagerly. */
- malloc_mutex_unlock(tsdn, &eset->mtx);
+ malloc_mutex_unlock(tsdn, &ecache->mtx);
arena_decay_extent(tsdn, arena, ehooks, edata);
return;
}
}
- extent_deactivate_locked(tsdn, arena, eset, edata);
+ extent_deactivate_locked(tsdn, arena, ecache, edata);
- malloc_mutex_unlock(tsdn, &eset->mtx);
+ malloc_mutex_unlock(tsdn, &ecache->mtx);
}
void
@@ -1312,7 +1313,8 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_gdump_sub(tsdn, edata);
}
- extent_record(tsdn, arena, ehooks, &arena->eset_retained, edata, false);
+ extent_record(tsdn, arena, ehooks, &arena->ecache_retained, edata,
+ false);
}
void
diff --git a/src/large.c b/src/large.c
index 67b47453..b8b70f4e 100644
--- a/src/large.c
+++ b/src/large.c
@@ -149,10 +149,10 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
bool commit = true;
edata_t *trail;
bool new_mapping;
- if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
+ if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->ecache_dirty,
edata_past_get(edata), trailsize, 0, CACHELINE, false, SC_NSIZES,
&is_zeroed_trail, &commit)) != NULL
- || (trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
+ || (trail = extents_alloc(tsdn, arena, ehooks, &arena->ecache_muzzy,
edata_past_get(edata), trailsize, 0, CACHELINE, false, SC_NSIZES,
&is_zeroed_trail, &commit)) != NULL) {
if (config_stats) {
diff --git a/test/unit/retained.c b/test/unit/retained.c
index 7993fd3d..e6b6f7b7 100644
--- a/test/unit/retained.c
+++ b/test/unit/retained.c
@@ -142,7 +142,7 @@ TEST_BEGIN(test_retained) {
size_t usable = 0;
size_t fragmented = 0;
for (pszind_t pind = sz_psz2ind(HUGEPAGE); pind <
- arena->extent_grow_next; pind++) {
+ arena->ecache_grow.next; pind++) {
size_t psz = sz_pind2sz(pind);
size_t psz_fragmented = psz % esz;
size_t psz_usable = psz - psz_fragmented;