Pull out edata_t caching into its own module.

Makefile.in

@@ -105,6 +105,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/ctl.c \
 	$(srcroot)src/div.c \
 	$(srcroot)src/edata.c \
+	$(srcroot)src/edata_cache.c \
 	$(srcroot)src/ehooks.c \
 	$(srcroot)src/eset.c \
 	$(srcroot)src/extent2.c \

include/jemalloc/internal/arena_structs.h

@@ -5,6 +5,7 @@
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/bin.h"
 #include "jemalloc/internal/bitmap.h"
+#include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/eset.h"
 #include "jemalloc/internal/extent_dss.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
@@ -184,15 +185,8 @@ struct arena_s {
 	pszind_t		retain_grow_limit;
 	malloc_mutex_t		extent_grow_mtx;
 
-	/*
+	/* The source of edata_t objects. */
-	 * Available edata structures that were allocated via
+	edata_cache_t		edata_cache;
-	 * base_alloc_edata().
-	 *
-	 * Synchronization: edata_avail_mtx.
-	 */
-	edata_tree_t		edata_avail;
-	atomic_zu_t		edata_avail_cnt;
-	malloc_mutex_t		edata_avail_mtx;
 
 	/*
 	 * bins is used to store heaps of free regions.

include/jemalloc/internal/edata_cache.h

@@ -0,0 +1,25 @@
+#ifndef JEMALLOC_INTERNAL_EDATA_CACHE_H
+#define JEMALLOC_INTERNAL_EDATA_CACHE_H
+
+/*
+ * A cache of edata_t structures allocated via base_alloc_edata (as opposed to
+ * the underlying extents they describe).  The contents of returned edata_t
+ * objects are garbage and cannot be relied upon.
+ */
+
+typedef struct edata_cache_s edata_cache_t;
+struct edata_cache_s {
+	edata_tree_t avail;
+	atomic_zu_t count;
+	malloc_mutex_t mtx;
+};
+
+bool edata_cache_init(edata_cache_t *edata_cache);
+edata_t *edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache,
+    base_t *base);
+void edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata);
+void edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache);
+void edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache);
+void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache);
+
+#endif /* JEMALLOC_INTERNAL_EDATA_CACHE_H */

include/jemalloc/internal/extent2.h

@@ -26,9 +26,6 @@ extern size_t opt_lg_extent_max_active_fit;
 
 extern rtree_t extents_rtree;
 
-edata_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
-void extent_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *edata);
-
 edata_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
     eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment,
     bool slab, szind_t szind, bool *zero, bool *commit);

include/jemalloc/internal/witness.h

@@ -43,7 +43,7 @@
 #define WITNESS_RANK_TCACHE_QL		13U
 #define WITNESS_RANK_EXTENT_GROW	14U
 #define WITNESS_RANK_EXTENTS		15U
-#define WITNESS_RANK_EDATA_AVAIL	16U
+#define WITNESS_RANK_EDATA_CACHE	16U
 
 #define WITNESS_RANK_EXTENT_POOL	17U
 #define WITNESS_RANK_RTREE		18U

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -45,6 +45,7 @@
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\div.c" />
     <ClCompile Include="..\..\..\..\src\edata.c" />
+    <ClCompile Include="..\..\..\..\src\edata_cache.c" />
     <ClCompile Include="..\..\..\..\src\ehooks.c" />
     <ClCompile Include="..\..\..\..\src\eset.c" />
     <ClCompile Include="..\..\..\..\src\extent2.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -45,6 +45,7 @@
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\div.c" />
     <ClCompile Include="..\..\..\..\src\edata.c" />
+    <ClCompile Include="..\..\..\..\src\edata_cache.c" />
     <ClCompile Include="..\..\..\..\src\ehooks.c" />
     <ClCompile Include="..\..\..\..\src\eset.c" />
     <ClCompile Include="..\..\..\..\src\extent2.c" />

src/arena.c

@@ -103,7 +103,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	    eset_npages_get(&arena->eset_retained) << LG_PAGE);
 
 	atomic_store_zu(&astats->edata_avail,
-	    atomic_load_zu(&arena->edata_avail_cnt, ATOMIC_RELAXED),
+	    atomic_load_zu(&arena->edata_cache.count, ATOMIC_RELAXED),
 	    ATOMIC_RELAXED);
 
 	arena_stats_accum_u64(&astats->decay_dirty.npurge,
@@ -224,7 +224,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 
 	/* Gather per arena mutex profiling data. */
 	READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
-	READ_ARENA_MUTEX_PROF_DATA(edata_avail_mtx,
+	READ_ARENA_MUTEX_PROF_DATA(edata_cache.mtx,
 	    arena_prof_mutex_extent_avail)
 	READ_ARENA_MUTEX_PROF_DATA(eset_dirty.mtx,
 	    arena_prof_mutex_extents_dirty)
@@ -2053,9 +2053,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 		goto label_error;
 	}
 
-	edata_avail_new(&arena->edata_avail);
+	if (edata_cache_init(&arena->edata_cache)) {
-	if (malloc_mutex_init(&arena->edata_avail_mtx, "edata_avail",
-	    WITNESS_RANK_EDATA_AVAIL, malloc_mutex_rank_exclusive)) {
 		goto label_error;
 	}
 
@@ -2201,7 +2199,7 @@ arena_prefork3(tsdn_t *tsdn, arena_t *arena) {
 
 void
 arena_prefork4(tsdn_t *tsdn, arena_t *arena) {
-	malloc_mutex_prefork(tsdn, &arena->edata_avail_mtx);
+	edata_cache_prefork(tsdn, &arena->edata_cache);
 }
 
 void
@@ -2235,7 +2233,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
 	}
 	malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
 	base_postfork_parent(tsdn, arena->base);
-	malloc_mutex_postfork_parent(tsdn, &arena->edata_avail_mtx);
+	edata_cache_postfork_parent(tsdn, &arena->edata_cache);
 	eset_postfork_parent(tsdn, &arena->eset_dirty);
 	eset_postfork_parent(tsdn, &arena->eset_muzzy);
 	eset_postfork_parent(tsdn, &arena->eset_retained);
@@ -2281,7 +2279,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
 	}
 	malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
 	base_postfork_child(tsdn, arena->base);
-	malloc_mutex_postfork_child(tsdn, &arena->edata_avail_mtx);
+	edata_cache_postfork_child(tsdn, &arena->edata_cache);
 	eset_postfork_child(tsdn, &arena->eset_dirty);
 	eset_postfork_child(tsdn, &arena->eset_muzzy);
 	eset_postfork_child(tsdn, &arena->eset_retained);

src/ctl.c

@@ -3010,7 +3010,7 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
 			continue;
 		}
 		MUTEX_PROF_RESET(arena->large_mtx);
-		MUTEX_PROF_RESET(arena->edata_avail_mtx);
+		MUTEX_PROF_RESET(arena->edata_cache.mtx);
 		MUTEX_PROF_RESET(arena->eset_dirty.mtx);
 		MUTEX_PROF_RESET(arena->eset_muzzy.mtx);
 		MUTEX_PROF_RESET(arena->eset_retained.mtx);

src/edata_cache.c

@@ -0,0 +1,47 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+bool
+edata_cache_init(edata_cache_t *edata_cache) {
+	if (malloc_mutex_init(&edata_cache->mtx, "edata_cache",
+	    WITNESS_RANK_EDATA_CACHE, malloc_mutex_rank_exclusive)) {
+		return true;
+	}
+
+	edata_avail_new(&edata_cache->avail);
+	return false;
+}
+
+edata_t *
+edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache, base_t *base) {
+	malloc_mutex_lock(tsdn, &edata_cache->mtx);
+	edata_t *edata = edata_avail_first(&edata_cache->avail);
+	if (edata == NULL) {
+		malloc_mutex_unlock(tsdn, &edata_cache->mtx);
+		return base_alloc_edata(tsdn, base);
+	}
+	edata_avail_remove(&edata_cache->avail, edata);
+	atomic_fetch_sub_zu(&edata_cache->count, 1, ATOMIC_RELAXED);
+	malloc_mutex_unlock(tsdn, &edata_cache->mtx);
+	return edata;
+}
+
+void
+edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata) {
+	malloc_mutex_lock(tsdn, &edata_cache->mtx);
+	edata_avail_insert(&edata_cache->avail, edata);
+	atomic_fetch_add_zu(&edata_cache->count, 1, ATOMIC_RELAXED);
+	malloc_mutex_unlock(tsdn, &edata_cache->mtx);
+}
+
+void edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache) {
+	malloc_mutex_prefork(tsdn, &edata_cache->mtx);
+}
+
+void edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache) {
+	malloc_mutex_postfork_parent(tsdn, &edata_cache->mtx);
+}
+
+void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache) {
+	malloc_mutex_postfork_child(tsdn, &edata_cache->mtx);
+}

src/extent2.c

@@ -163,28 +163,6 @@ extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
 	}
 }
 
-edata_t *
-extent_alloc(tsdn_t *tsdn, arena_t *arena) {
-	malloc_mutex_lock(tsdn, &arena->edata_avail_mtx);
-	edata_t *edata = edata_avail_first(&arena->edata_avail);
-	if (edata == NULL) {
-		malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
-		return base_alloc_edata(tsdn, arena->base);
-	}
-	edata_avail_remove(&arena->edata_avail, edata);
-	atomic_fetch_sub_zu(&arena->edata_avail_cnt, 1, ATOMIC_RELAXED);
-	malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
-	return edata;
-}
-
-void
-extent_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
-	malloc_mutex_lock(tsdn, &arena->edata_avail_mtx);
-	edata_avail_insert(&arena->edata_avail, edata);
-	atomic_fetch_add_zu(&arena->edata_avail_cnt, 1, ATOMIC_RELAXED);
-	malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
-}
-
 static bool
 extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
     rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata) {
@@ -317,7 +295,7 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
 			    edata_size_get(edata), growing_retained);
 		}
 	}
-	extent_dalloc(tsdn, arena, edata);
+	edata_cache_put(tsdn, &arena->edata_cache, edata);
 }
 
 static void
@@ -858,7 +836,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 		alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
 	}
 
-	edata_t *edata = extent_alloc(tsdn, arena);
+	edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache,
+	    arena->base);
 	if (edata == NULL) {
 		goto label_err;
 	}
@@ -872,12 +851,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 	    SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed,
 	    committed, true, EXTENT_IS_HEAD);
 	if (ptr == NULL) {
-		extent_dalloc(tsdn, arena, edata);
+		edata_cache_put(tsdn, &arena->edata_cache, edata);
 		goto label_err;
 	}
 
 	if (extent_register_no_gdump_add(tsdn, edata)) {
-		extent_dalloc(tsdn, arena, edata);
+		edata_cache_put(tsdn, &arena->edata_cache, edata);
 		goto label_err;
 	}
 
@@ -1021,7 +1000,8 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
     void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
     szind_t szind, bool *zero, bool *commit) {
 	size_t esize = size + pad;
-	edata_t *edata = extent_alloc(tsdn, arena);
+	edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache,
+	    arena->base);
 	if (edata == NULL) {
 		return NULL;
 	}
@@ -1029,7 +1009,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 	void *addr = ehooks_alloc(tsdn, ehooks, new_addr, esize, palignment,
 	    zero, commit, arena_ind_get(arena));
 	if (addr == NULL) {
-		extent_dalloc(tsdn, arena, edata);
+		edata_cache_put(tsdn, &arena->edata_cache, edata);
 		return NULL;
 	}
 	edata_init(edata, arena_ind_get(arena), addr, esize, slab, szind,
@@ -1039,7 +1019,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 		extent_addr_randomize(tsdn, arena, edata, alignment);
 	}
 	if (extent_register(tsdn, edata)) {
-		extent_dalloc(tsdn, arena, edata);
+		edata_cache_put(tsdn, &arena->edata_cache, edata);
 		return NULL;
 	}
 
@@ -1257,7 +1237,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
 	    WITNESS_RANK_CORE, 0);
 
 	if (extent_register(tsdn, edata)) {
-		extent_dalloc(tsdn, arena, edata);
+		edata_cache_put(tsdn, &arena->edata_cache, edata);
 		return;
 	}
 	extent_dalloc_wrapper(tsdn, arena, ehooks, edata);
@@ -1287,7 +1267,7 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 	    arena_ind_get(arena));
 
 	if (!err) {
-		extent_dalloc(tsdn, arena, edata);
+		edata_cache_put(tsdn, &arena->edata_cache, edata);
 	}
 
 	return err;
@@ -1359,7 +1339,7 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 	    edata_size_get(edata), edata_committed_get(edata),
 	    arena_ind_get(arena));
 
-	extent_dalloc(tsdn, arena, edata);
+	edata_cache_put(tsdn, &arena->edata_cache, edata);
 }
 
 static bool
@@ -1445,7 +1425,8 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 		return NULL;
 	}
 
-	edata_t *trail = extent_alloc(tsdn, arena);
+	edata_t *trail = edata_cache_get(tsdn, &arena->edata_cache,
+	    arena->base);
 	if (trail == NULL) {
 		goto label_error_a;
 	}
@@ -1505,7 +1486,7 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 label_error_c:
 	extent_unlock_edata2(tsdn, edata, trail);
 label_error_b:
-	extent_dalloc(tsdn, arena, trail);
+	edata_cache_put(tsdn, &arena->edata_cache, trail);
 label_error_a:
 	return NULL;
 }
@@ -1611,7 +1592,7 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a,
 	 * arena (i.e. this one).
 	 */
 	assert(edata_arena_ind_get(b) == arena_ind_get(arena));
-	extent_dalloc(tsdn, arena, b);
+	edata_cache_put(tsdn, &arena->edata_cache, b);
 
 	return false;
 }

src/extent_dss.c

@@ -123,7 +123,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
 		return NULL;
 	}
 
-	gap = extent_alloc(tsdn, arena);
+	gap = edata_cache_get(tsdn, &arena->edata_cache, arena->base);
 	if (gap == NULL) {
 		return NULL;
 	}
@@ -188,7 +188,8 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
 				if (gap_size_page != 0) {
 					extent_dalloc_gap(tsdn, arena, gap);
 				} else {
-					extent_dalloc(tsdn, arena, gap);
+					edata_cache_put(tsdn,
+					    &arena->edata_cache, gap);
 				}
 				if (!*commit) {
 					*commit = pages_decommit(ret, size);
@@ -224,7 +225,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
 	}
 label_oom:
 	extent_dss_extending_finish();
-	extent_dalloc(tsdn, arena, gap);
+	edata_cache_put(tsdn, &arena->edata_cache, gap);
 	return NULL;
 }