diff --git a/Makefile.in b/Makefile.in
index 86a51ccb..f75ae4b6 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -105,6 +105,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/ctl.c \
$(srcroot)src/div.c \
$(srcroot)src/edata.c \
+ $(srcroot)src/edata_cache.c \
$(srcroot)src/ehooks.c \
$(srcroot)src/eset.c \
$(srcroot)src/extent2.c \
diff --git a/include/jemalloc/internal/arena_structs.h b/include/jemalloc/internal/arena_structs.h
index aac620b9..38c8b27c 100644
--- a/include/jemalloc/internal/arena_structs.h
+++ b/include/jemalloc/internal/arena_structs.h
@@ -5,6 +5,7 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/bitmap.h"
+#include "jemalloc/internal/edata_cache.h"
#include "jemalloc/internal/eset.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
@@ -184,15 +185,8 @@ struct arena_s {
pszind_t retain_grow_limit;
malloc_mutex_t extent_grow_mtx;
- /*
- * Available edata structures that were allocated via
- * base_alloc_edata().
- *
- * Synchronization: edata_avail_mtx.
- */
- edata_tree_t edata_avail;
- atomic_zu_t edata_avail_cnt;
- malloc_mutex_t edata_avail_mtx;
+ /* The source of edata_t objects. */
+ edata_cache_t edata_cache;
/*
* bins is used to store heaps of free regions.
diff --git a/include/jemalloc/internal/edata_cache.h b/include/jemalloc/internal/edata_cache.h
new file mode 100644
index 00000000..fc184084
--- /dev/null
+++ b/include/jemalloc/internal/edata_cache.h
@@ -0,0 +1,25 @@
+#ifndef JEMALLOC_INTERNAL_EDATA_CACHE_H
+#define JEMALLOC_INTERNAL_EDATA_CACHE_H
+
+/*
+ * A cache of edata_t structures allocated via base_alloc_edata (as opposed to
+ * the underlying extents they describe). The contents of returned edata_t
+ * objects are garbage and cannot be relied upon.
+ */
+
+typedef struct edata_cache_s edata_cache_t;
+struct edata_cache_s {
+ edata_tree_t avail;
+ atomic_zu_t count;
+ malloc_mutex_t mtx;
+};
+
+bool edata_cache_init(edata_cache_t *edata_cache);
+edata_t *edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache,
+ base_t *base);
+void edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata);
+void edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache);
+void edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache);
+void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache);
+
+#endif /* JEMALLOC_INTERNAL_EDATA_CACHE_H */
diff --git a/include/jemalloc/internal/extent2.h b/include/jemalloc/internal/extent2.h
index ef232677..629474ee 100644
--- a/include/jemalloc/internal/extent2.h
+++ b/include/jemalloc/internal/extent2.h
@@ -26,9 +26,6 @@ extern size_t opt_lg_extent_max_active_fit;
extern rtree_t extents_rtree;
-edata_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
-void extent_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *edata);
-
edata_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit);
diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h
index ddbcf9d2..985e0a33 100644
--- a/include/jemalloc/internal/witness.h
+++ b/include/jemalloc/internal/witness.h
@@ -43,7 +43,7 @@
#define WITNESS_RANK_TCACHE_QL 13U
#define WITNESS_RANK_EXTENT_GROW 14U
#define WITNESS_RANK_EXTENTS 15U
-#define WITNESS_RANK_EDATA_AVAIL 16U
+#define WITNESS_RANK_EDATA_CACHE 16U
#define WITNESS_RANK_EXTENT_POOL 17U
#define WITNESS_RANK_RTREE 18U
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index 9dfc36d2..23312d3b 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -45,6 +45,7 @@
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index 0ec4d1ee..76c16c5a 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -45,6 +45,7 @@
+
diff --git a/src/arena.c b/src/arena.c
index f05a1d17..a23419a3 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -103,7 +103,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
eset_npages_get(&arena->eset_retained) << LG_PAGE);
atomic_store_zu(&astats->edata_avail,
- atomic_load_zu(&arena->edata_avail_cnt, ATOMIC_RELAXED),
+ atomic_load_zu(&arena->edata_cache.count, ATOMIC_RELAXED),
ATOMIC_RELAXED);
arena_stats_accum_u64(&astats->decay_dirty.npurge,
@@ -224,7 +224,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
/* Gather per arena mutex profiling data. */
READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
- READ_ARENA_MUTEX_PROF_DATA(edata_avail_mtx,
+ READ_ARENA_MUTEX_PROF_DATA(edata_cache.mtx,
arena_prof_mutex_extent_avail)
READ_ARENA_MUTEX_PROF_DATA(eset_dirty.mtx,
arena_prof_mutex_extents_dirty)
@@ -2053,9 +2053,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
goto label_error;
}
- edata_avail_new(&arena->edata_avail);
- if (malloc_mutex_init(&arena->edata_avail_mtx, "edata_avail",
- WITNESS_RANK_EDATA_AVAIL, malloc_mutex_rank_exclusive)) {
+ if (edata_cache_init(&arena->edata_cache)) {
goto label_error;
}
@@ -2201,7 +2199,7 @@ arena_prefork3(tsdn_t *tsdn, arena_t *arena) {
void
arena_prefork4(tsdn_t *tsdn, arena_t *arena) {
- malloc_mutex_prefork(tsdn, &arena->edata_avail_mtx);
+ edata_cache_prefork(tsdn, &arena->edata_cache);
}
void
@@ -2235,7 +2233,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
}
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
base_postfork_parent(tsdn, arena->base);
- malloc_mutex_postfork_parent(tsdn, &arena->edata_avail_mtx);
+ edata_cache_postfork_parent(tsdn, &arena->edata_cache);
eset_postfork_parent(tsdn, &arena->eset_dirty);
eset_postfork_parent(tsdn, &arena->eset_muzzy);
eset_postfork_parent(tsdn, &arena->eset_retained);
@@ -2281,7 +2279,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
}
malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
base_postfork_child(tsdn, arena->base);
- malloc_mutex_postfork_child(tsdn, &arena->edata_avail_mtx);
+ edata_cache_postfork_child(tsdn, &arena->edata_cache);
eset_postfork_child(tsdn, &arena->eset_dirty);
eset_postfork_child(tsdn, &arena->eset_muzzy);
eset_postfork_child(tsdn, &arena->eset_retained);
diff --git a/src/ctl.c b/src/ctl.c
index 1e72bf4c..a58b22ba 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -3010,7 +3010,7 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
continue;
}
MUTEX_PROF_RESET(arena->large_mtx);
- MUTEX_PROF_RESET(arena->edata_avail_mtx);
+ MUTEX_PROF_RESET(arena->edata_cache.mtx);
MUTEX_PROF_RESET(arena->eset_dirty.mtx);
MUTEX_PROF_RESET(arena->eset_muzzy.mtx);
MUTEX_PROF_RESET(arena->eset_retained.mtx);
diff --git a/src/edata_cache.c b/src/edata_cache.c
new file mode 100644
index 00000000..4d026029
--- /dev/null
+++ b/src/edata_cache.c
@@ -0,0 +1,47 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+bool
+edata_cache_init(edata_cache_t *edata_cache) {
+ if (malloc_mutex_init(&edata_cache->mtx, "edata_cache",
+ WITNESS_RANK_EDATA_CACHE, malloc_mutex_rank_exclusive)) {
+ return true;
+ }
+
+ edata_avail_new(&edata_cache->avail);
+ return false;
+}
+
+edata_t *
+edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache, base_t *base) {
+ malloc_mutex_lock(tsdn, &edata_cache->mtx);
+ edata_t *edata = edata_avail_first(&edata_cache->avail);
+ if (edata == NULL) {
+ malloc_mutex_unlock(tsdn, &edata_cache->mtx);
+ return base_alloc_edata(tsdn, base);
+ }
+ edata_avail_remove(&edata_cache->avail, edata);
+ atomic_fetch_sub_zu(&edata_cache->count, 1, ATOMIC_RELAXED);
+ malloc_mutex_unlock(tsdn, &edata_cache->mtx);
+ return edata;
+}
+
+void
+edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata) {
+ malloc_mutex_lock(tsdn, &edata_cache->mtx);
+ edata_avail_insert(&edata_cache->avail, edata);
+ atomic_fetch_add_zu(&edata_cache->count, 1, ATOMIC_RELAXED);
+ malloc_mutex_unlock(tsdn, &edata_cache->mtx);
+}
+
+void edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache) {
+ malloc_mutex_prefork(tsdn, &edata_cache->mtx);
+}
+
+void edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache) {
+ malloc_mutex_postfork_parent(tsdn, &edata_cache->mtx);
+}
+
+void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache) {
+ malloc_mutex_postfork_child(tsdn, &edata_cache->mtx);
+}
diff --git a/src/extent2.c b/src/extent2.c
index 5bacb8fe..b77e4b89 100644
--- a/src/extent2.c
+++ b/src/extent2.c
@@ -163,28 +163,6 @@ extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
}
}
-edata_t *
-extent_alloc(tsdn_t *tsdn, arena_t *arena) {
- malloc_mutex_lock(tsdn, &arena->edata_avail_mtx);
- edata_t *edata = edata_avail_first(&arena->edata_avail);
- if (edata == NULL) {
- malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
- return base_alloc_edata(tsdn, arena->base);
- }
- edata_avail_remove(&arena->edata_avail, edata);
- atomic_fetch_sub_zu(&arena->edata_avail_cnt, 1, ATOMIC_RELAXED);
- malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
- return edata;
-}
-
-void
-extent_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
- malloc_mutex_lock(tsdn, &arena->edata_avail_mtx);
- edata_avail_insert(&arena->edata_avail, edata);
- atomic_fetch_add_zu(&arena->edata_avail_cnt, 1, ATOMIC_RELAXED);
- malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
-}
-
static bool
extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata) {
@@ -317,7 +295,7 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
edata_size_get(edata), growing_retained);
}
}
- extent_dalloc(tsdn, arena, edata);
+ edata_cache_put(tsdn, &arena->edata_cache, edata);
}
static void
@@ -858,7 +836,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
}
- edata_t *edata = extent_alloc(tsdn, arena);
+ edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache,
+ arena->base);
if (edata == NULL) {
goto label_err;
}
@@ -872,12 +851,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed,
committed, true, EXTENT_IS_HEAD);
if (ptr == NULL) {
- extent_dalloc(tsdn, arena, edata);
+ edata_cache_put(tsdn, &arena->edata_cache, edata);
goto label_err;
}
if (extent_register_no_gdump_add(tsdn, edata)) {
- extent_dalloc(tsdn, arena, edata);
+ edata_cache_put(tsdn, &arena->edata_cache, edata);
goto label_err;
}
@@ -1021,7 +1000,8 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit) {
size_t esize = size + pad;
- edata_t *edata = extent_alloc(tsdn, arena);
+ edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache,
+ arena->base);
if (edata == NULL) {
return NULL;
}
@@ -1029,7 +1009,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *addr = ehooks_alloc(tsdn, ehooks, new_addr, esize, palignment,
zero, commit, arena_ind_get(arena));
if (addr == NULL) {
- extent_dalloc(tsdn, arena, edata);
+ edata_cache_put(tsdn, &arena->edata_cache, edata);
return NULL;
}
edata_init(edata, arena_ind_get(arena), addr, esize, slab, szind,
@@ -1039,7 +1019,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_addr_randomize(tsdn, arena, edata, alignment);
}
if (extent_register(tsdn, edata)) {
- extent_dalloc(tsdn, arena, edata);
+ edata_cache_put(tsdn, &arena->edata_cache, edata);
return NULL;
}
@@ -1257,7 +1237,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
WITNESS_RANK_CORE, 0);
if (extent_register(tsdn, edata)) {
- extent_dalloc(tsdn, arena, edata);
+ edata_cache_put(tsdn, &arena->edata_cache, edata);
return;
}
extent_dalloc_wrapper(tsdn, arena, ehooks, edata);
@@ -1287,7 +1267,7 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
arena_ind_get(arena));
if (!err) {
- extent_dalloc(tsdn, arena, edata);
+ edata_cache_put(tsdn, &arena->edata_cache, edata);
}
return err;
@@ -1359,7 +1339,7 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
edata_size_get(edata), edata_committed_get(edata),
arena_ind_get(arena));
- extent_dalloc(tsdn, arena, edata);
+ edata_cache_put(tsdn, &arena->edata_cache, edata);
}
static bool
@@ -1445,7 +1425,8 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
return NULL;
}
- edata_t *trail = extent_alloc(tsdn, arena);
+ edata_t *trail = edata_cache_get(tsdn, &arena->edata_cache,
+ arena->base);
if (trail == NULL) {
goto label_error_a;
}
@@ -1505,7 +1486,7 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
label_error_c:
extent_unlock_edata2(tsdn, edata, trail);
label_error_b:
- extent_dalloc(tsdn, arena, trail);
+ edata_cache_put(tsdn, &arena->edata_cache, trail);
label_error_a:
return NULL;
}
@@ -1611,7 +1592,7 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a,
* arena (i.e. this one).
*/
assert(edata_arena_ind_get(b) == arena_ind_get(arena));
- extent_dalloc(tsdn, arena, b);
+ edata_cache_put(tsdn, &arena->edata_cache, b);
return false;
}
diff --git a/src/extent_dss.c b/src/extent_dss.c
index a66afb68..25ba944b 100644
--- a/src/extent_dss.c
+++ b/src/extent_dss.c
@@ -123,7 +123,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
return NULL;
}
- gap = extent_alloc(tsdn, arena);
+ gap = edata_cache_get(tsdn, &arena->edata_cache, arena->base);
if (gap == NULL) {
return NULL;
}
@@ -188,7 +188,8 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
if (gap_size_page != 0) {
extent_dalloc_gap(tsdn, arena, gap);
} else {
- extent_dalloc(tsdn, arena, gap);
+ edata_cache_put(tsdn,
+ &arena->edata_cache, gap);
}
if (!*commit) {
*commit = pages_decommit(ret, size);
@@ -224,7 +225,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
}
label_oom:
extent_dss_extending_finish();
- extent_dalloc(tsdn, arena, gap);
+ edata_cache_put(tsdn, &arena->edata_cache, gap);
return NULL;
}