Fix extent_grow_next management.

Fix management of extent_grow_next to serialize operations that may grow
retained memory.  This assures that the sizes of the newly allocated
extents correspond to the size classes in the intended growth sequence.

Fix management of extent_grow_next to skip size classes if a request is
too large to be satisfied by the next size in the growth sequence.  This
avoids the potential for an arbitrary number of requests to bypass
triggering extent_grow_next increases.

This resolves #858.

include/jemalloc/internal/arena_structs_b.h

@@ -238,9 +238,10 @@ struct arena_s {
 	 * be effective even if multiple arenas' extent allocation requests are
 	 * highly interleaved.
 	 *
-	 * Synchronization: atomic.
+	 * Synchronization: extent_grow_mtx
 	 */
-	atomic_u_t		extent_grow_next;
+	pszind_t		extent_grow_next;
+	malloc_mutex_t		extent_grow_mtx;
 
 	/*
 	 * Available extent structures that were allocated via

include/jemalloc/internal/witness.h

@@ -41,13 +41,14 @@
 
 #define WITNESS_RANK_DECAY		11U
 #define WITNESS_RANK_TCACHE_QL		12U
-#define WITNESS_RANK_EXTENTS		13U
+#define WITNESS_RANK_EXTENT_GROW	13U
-#define WITNESS_RANK_EXTENT_FREELIST	14U
+#define WITNESS_RANK_EXTENTS		14U
+#define WITNESS_RANK_EXTENT_AVAIL	15U
 
-#define WITNESS_RANK_EXTENT_POOL	15U
+#define WITNESS_RANK_EXTENT_POOL	16U
-#define WITNESS_RANK_RTREE		16U
+#define WITNESS_RANK_RTREE		17U
-#define WITNESS_RANK_BASE		17U
+#define WITNESS_RANK_BASE		18U
-#define WITNESS_RANK_ARENA_LARGE	18U
+#define WITNESS_RANK_ARENA_LARGE	19U
 
 #define WITNESS_RANK_LEAF		0xffffffffU
 #define WITNESS_RANK_ARENA_BIN		WITNESS_RANK_LEAF

src/arena.c

@@ -2001,14 +2001,15 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 		goto label_error;
 	}
 
-	if (opt_retain) {
+	arena->extent_grow_next = psz2ind(HUGEPAGE);
-		atomic_store_u(&arena->extent_grow_next, psz2ind(HUGEPAGE),
+	if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow",
-		    ATOMIC_RELAXED);
+	    WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
+		goto label_error;
 	}
 
 	extent_avail_new(&arena->extent_avail);
 	if (malloc_mutex_init(&arena->extent_avail_mtx, "extent_avail",
-	    WITNESS_RANK_EXTENT_FREELIST, malloc_mutex_rank_exclusive)) {
+	    WITNESS_RANK_EXTENT_AVAIL, malloc_mutex_rank_exclusive)) {
 		goto label_error;
 	}
 

src/extent.c

@@ -16,8 +16,8 @@ mutex_pool_t	extent_mutex_pool;
 static const bitmap_info_t extents_bitmap_info =
     BITMAP_INFO_INITIALIZER(NPSIZES+1);
 
-static void	*extent_alloc_default(extent_hooks_t *extent_hooks,
+static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr,
-    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
+    size_t size, size_t alignment, bool *zero, bool *commit,
     unsigned arena_ind);
 static bool extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr,
     size_t size, bool committed, unsigned arena_ind);
@@ -25,24 +25,42 @@ static void	extent_destroy_default(extent_hooks_t *extent_hooks, void *addr,
     size_t size, bool committed, unsigned arena_ind);
 static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr,
     size_t size, size_t offset, size_t length, unsigned arena_ind);
+static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length, bool growing_retained);
 static bool extent_decommit_default(extent_hooks_t *extent_hooks,
     void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
 #ifdef PAGES_CAN_PURGE_LAZY
-static bool	extent_purge_lazy_default(extent_hooks_t *extent_hooks,
+static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr,
-    void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
+    size_t size, size_t offset, size_t length, unsigned arena_ind);
 #endif
+static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length, bool growing_retained);
 #ifdef PAGES_CAN_PURGE_FORCED
 static bool extent_purge_forced_default(extent_hooks_t *extent_hooks,
     void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
 #endif
+static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length, bool growing_retained);
 #ifdef JEMALLOC_MAPS_COALESCE
 static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr,
     size_t size, size_t size_a, size_t size_b, bool committed,
     unsigned arena_ind);
+#endif
+static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
+    szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
+    bool growing_retained);
+#ifdef JEMALLOC_MAPS_COALESCE
 static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a,
     size_t size_a, void *addr_b, size_t size_b, bool committed,
     unsigned arena_ind);
 #endif
+static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b,
+    bool growing_retained);
 
 const extent_hooks_t	extent_hooks_default = {
 	extent_alloc_default,
@@ -85,12 +103,13 @@ static void extent_deregister(tsdn_t *tsdn, extent_t *extent);
 static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr,
     size_t usize, size_t pad, size_t alignment, bool slab, szind_t szind,
-    bool *zero, bool *commit);
+    bool *zero, bool *commit, bool growing_retained);
 static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents,
-    extent_t *extent, bool *coalesced);
+    extent_t *extent, bool *coalesced, bool growing_retained);
 static void extent_record(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent);
+    extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent,
+    bool growing_retained);
 
 /******************************************************************************/
 
@@ -150,9 +169,6 @@ extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) {
 
 extent_t *
 extent_alloc(tsdn_t *tsdn, arena_t *arena) {
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
-
 	malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
 	extent_t *extent = extent_avail_first(&arena->extent_avail);
 	if (extent == NULL) {
@@ -166,9 +182,6 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena) {
 
 void
 extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
-
 	malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
 	extent_avail_insert(&arena->extent_avail, extent);
 	malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
@@ -402,7 +415,7 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena,
 	extent_state_set(extent, extent_state_active);
 	bool coalesced;
 	extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx,
-	    extents, extent, &coalesced);
+	    extents, extent, &coalesced, false);
 	extent_state_set(extent, extents_state_get(extents));
 
 	if (!coalesced) {
@@ -422,7 +435,7 @@ extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 	    WITNESS_RANK_CORE, 0);
 
 	return extent_recycle(tsdn, arena, r_extent_hooks, extents, new_addr,
-	    size, pad, alignment, slab, szind, zero, commit);
+	    size, pad, alignment, slab, szind, zero, commit, false);
 }
 
 void
@@ -436,7 +449,7 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 	extent_addr_set(extent, extent_base_get(extent));
 	extent_zeroed_set(extent, false);
 
-	extent_record(tsdn, arena, r_extent_hooks, extents, extent);
+	extent_record(tsdn, arena, r_extent_hooks, extents, extent, false);
 }
 
 extent_t *
@@ -506,16 +519,17 @@ label_return:
 
 static void
 extents_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
-    extents_t *extents, extent_t *extent) {
+    extents_t *extents, extent_t *extent, bool growing_retained) {
 	/*
 	 * Leak extent after making sure its pages have already been purged, so
 	 * that this is only a virtual memory leak.
 	 */
 	if (extents_state_get(extents) == extent_state_dirty) {
-		if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks,
+		if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks,
-		    extent, 0, extent_size_get(extent))) {
+		    extent, 0, extent_size_get(extent), growing_retained)) {
-			extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks,
+			extent_purge_forced_impl(tsdn, arena, r_extent_hooks,
-			    extent, 0, extent_size_get(extent));
+			    extent, 0, extent_size_get(extent),
+			    growing_retained);
 		}
 	}
 	extent_dalloc(tsdn, arena, extent);
@@ -734,13 +748,10 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent) {
 static extent_t *
 extent_recycle_extract(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents,
-    bool locked, void *new_addr, size_t size, size_t pad, size_t alignment,
+    void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
-    bool slab, bool *zero, bool *commit) {
+    bool *zero, bool *commit, bool growing_retained) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, locked ? 1 : 0);
+	    WITNESS_RANK_CORE, growing_retained ? 1 : 0);
-	if (locked) {
-		malloc_mutex_assert_owner(tsdn, &extents->mtx);
-	}
 	assert(alignment > 0);
 	if (config_debug && new_addr != NULL) {
 		/*
@@ -765,9 +776,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena,
 	if (alloc_size < esize) {
 		return NULL;
 	}
-	if (!locked) {
 	malloc_mutex_lock(tsdn, &extents->mtx);
-	}
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
 	extent_t *extent;
 	if (new_addr != NULL) {
@@ -791,16 +800,12 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena,
 		extent = extents_fit_locked(tsdn, arena, extents, alloc_size);
 	}
 	if (extent == NULL) {
-		if (!locked) {
 		malloc_mutex_unlock(tsdn, &extents->mtx);
-		}
 		return NULL;
 	}
 
 	extent_activate_locked(tsdn, arena, extents, extent, false);
-	if (!locked) {
 	malloc_mutex_unlock(tsdn, &extents->mtx);
-	}
 
 	if (extent_zeroed_get(extent)) {
 		*zero = true;
@@ -816,7 +821,7 @@ static extent_t *
 extent_recycle_split(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents,
     void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
-    szind_t szind, extent_t *extent) {
+    szind_t szind, extent_t *extent, bool growing_retained) {
 	size_t esize = size + pad;
 	size_t leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(extent),
 	    PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(extent);
@@ -827,13 +832,13 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena,
 	/* Split the lead. */
 	if (leadsize != 0) {
 		extent_t *lead = extent;
-		extent = extent_split_wrapper(tsdn, arena, r_extent_hooks,
+		extent = extent_split_impl(tsdn, arena, r_extent_hooks,
 		    lead, leadsize, NSIZES, false, esize + trailsize, szind,
-		    slab);
+		    slab, growing_retained);
 		if (extent == NULL) {
 			extent_deregister(tsdn, lead);
 			extents_leak(tsdn, arena, r_extent_hooks, extents,
-			    lead);
+			    lead, growing_retained);
 			return NULL;
 		}
 		extent_deactivate(tsdn, arena, extents, lead, false);
@@ -841,13 +846,13 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena,
 
 	/* Split the trail. */
 	if (trailsize != 0) {
-		extent_t *trail = extent_split_wrapper(tsdn, arena,
+		extent_t *trail = extent_split_impl(tsdn, arena,
 		    r_extent_hooks, extent, esize, szind, slab, trailsize,
-		    NSIZES, false);
+		    NSIZES, false, growing_retained);
 		if (trail == NULL) {
 			extent_deregister(tsdn, extent);
 			extents_leak(tsdn, arena, r_extent_hooks, extents,
-			    extent);
+			    extent, growing_retained);
 			return NULL;
 		}
 		extent_deactivate(tsdn, arena, extents, trail, false);
@@ -875,9 +880,10 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena,
 static extent_t *
 extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
     extents_t *extents, void *new_addr, size_t size, size_t pad,
-    size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
+    size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit,
+    bool growing_retained) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
+	    WITNESS_RANK_CORE, growing_retained ? 1 : 0);
 	assert(new_addr == NULL || !slab);
 	assert(pad == 0 || !slab);
 	assert(!*zero || !slab);
@@ -887,8 +893,8 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 
 	bool committed = false;
 	extent_t *extent = extent_recycle_extract(tsdn, arena, r_extent_hooks,
-	    rtree_ctx, extents, false, new_addr, size, pad, alignment, slab,
+	    rtree_ctx, extents, new_addr, size, pad, alignment, slab, zero,
-	    zero, &committed);
+	    &committed, growing_retained);
 	if (extent == NULL) {
 		return NULL;
 	}
@@ -897,16 +903,17 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 	}
 
 	extent = extent_recycle_split(tsdn, arena, r_extent_hooks, rtree_ctx,
-	    extents, new_addr, size, pad, alignment, slab, szind, extent);
+	    extents, new_addr, size, pad, alignment, slab, szind, extent,
+	    growing_retained);
 	if (extent == NULL) {
 		return NULL;
 	}
 
 	if (*commit && !extent_committed_get(extent)) {
-		if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent,
+		if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent,
-		    0, extent_size_get(extent))) {
+		    0, extent_size_get(extent), growing_retained)) {
 			extent_record(tsdn, arena, r_extent_hooks, extents,
-			    extent);
+			    extent, growing_retained);
 			return NULL;
 		}
 		extent_zeroed_set(extent, true);
@@ -1012,35 +1019,41 @@ extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
  */
 static extent_t *
 extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad,
+    extent_hooks_t **r_extent_hooks, size_t size, size_t pad, size_t alignment,
-    size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
+    bool slab, szind_t szind, bool *zero, bool *commit) {
+	malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx);
 	assert(pad == 0 || !slab);
 	assert(!*zero || !slab);
 
-	/*
-	 * Check whether the next extent size in the series would be large
-	 * enough to satisfy this request.  If no, just bail, so that e.g. a
-	 * series of unsatisfiable allocation requests doesn't cause unused
-	 * extent creation as a side effect.
-	 */
 	size_t esize = size + pad;
-	size_t alloc_size = pind2sz(atomic_load_u(&arena->extent_grow_next,
-	    ATOMIC_RELAXED));
 	size_t alloc_size_min = esize + PAGE_CEILING(alignment) - PAGE;
 	/* Beware size_t wrap-around. */
 	if (alloc_size_min < esize) {
-		return NULL;
+		goto label_err;
 	}
-	if (alloc_size < alloc_size_min) {
+	/*
-		return NULL;
+	 * Find the next extent size in the series that would be large enough to
+	 * satisfy this request.
+	 */
+	pszind_t egn_skip = 0;
+	size_t alloc_size = pind2sz(arena->extent_grow_next + egn_skip);
+	while (alloc_size < alloc_size_min) {
+		egn_skip++;
+		if (arena->extent_grow_next + egn_skip == NPSIZES) {
+			/* Outside legal range. */
+			goto label_err;
 		}
+		assert(arena->extent_grow_next + egn_skip < NPSIZES);
+		alloc_size = pind2sz(arena->extent_grow_next + egn_skip);
+	}
+
 	extent_t *extent = extent_alloc(tsdn, arena);
 	if (extent == NULL) {
-		return NULL;
+		goto label_err;
 	}
 	bool zeroed = false;
 	bool committed = false;
-	void *ptr = extent_alloc_core(tsdn, arena, new_addr, alloc_size, PAGE,
+	void *ptr = extent_alloc_core(tsdn, arena, NULL, alloc_size, PAGE,
 	    &zeroed, &committed, (dss_prec_t)atomic_load_u(&arena->dss_prec,
 	    ATOMIC_RELAXED));
 	extent_init(extent, arena, ptr, alloc_size, false, NSIZES,
@@ -1048,17 +1061,16 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
 	    committed);
 	if (ptr == NULL) {
 		extent_dalloc(tsdn, arena, extent);
-		return NULL;
+		goto label_err;
 	}
 	if (extent_register_no_gdump_add(tsdn, extent)) {
 		extents_leak(tsdn, arena, r_extent_hooks,
-		    &arena->extents_retained, extent);
+		    &arena->extents_retained, extent, true);
-		return NULL;
+		goto label_err;
 	}
 
 	size_t leadsize = ALIGNMENT_CEILING((uintptr_t)ptr,
 	    PAGE_CEILING(alignment)) - (uintptr_t)ptr;
-	assert(new_addr == NULL || leadsize == 0);
 	assert(alloc_size >= leadsize + esize);
 	size_t trailsize = alloc_size - leadsize - esize;
 	if (extent_zeroed_get(extent) && extent_committed_get(extent)) {
@@ -1071,31 +1083,31 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
 	/* Split the lead. */
 	if (leadsize != 0) {
 		extent_t *lead = extent;
-		extent = extent_split_wrapper(tsdn, arena, r_extent_hooks, lead,
+		extent = extent_split_impl(tsdn, arena, r_extent_hooks, lead,
-		    leadsize, NSIZES, false, esize + trailsize, szind, slab);
+		    leadsize, NSIZES, false, esize + trailsize, szind, slab,
+		    true);
 		if (extent == NULL) {
 			extent_deregister(tsdn, lead);
 			extents_leak(tsdn, arena, r_extent_hooks,
-			    &arena->extents_retained, lead);
+			    &arena->extents_retained, lead, true);
-			return NULL;
+			goto label_err;
 		}
 		extent_record(tsdn, arena, r_extent_hooks,
-		    &arena->extents_retained, lead);
+		    &arena->extents_retained, lead, true);
 	}
 
 	/* Split the trail. */
 	if (trailsize != 0) {
-		extent_t *trail = extent_split_wrapper(tsdn, arena,
+		extent_t *trail = extent_split_impl(tsdn, arena, r_extent_hooks,
-		    r_extent_hooks, extent, esize, szind, slab, trailsize,
+		    extent, esize, szind, slab, trailsize, NSIZES, false, true);
-		    NSIZES, false);
 		if (trail == NULL) {
 			extent_deregister(tsdn, extent);
 			extents_leak(tsdn, arena, r_extent_hooks,
-			    &arena->extents_retained, extent);
+			    &arena->extents_retained, extent, true);
-			return NULL;
+			goto label_err;
 		}
 		extent_record(tsdn, arena, r_extent_hooks,
-		    &arena->extents_retained, trail);
+		    &arena->extents_retained, trail, true);
 	} else if (leadsize == 0) {
 		/*
 		 * Splitting causes szind to be set as a side effect, but no
@@ -1119,15 +1131,27 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
 	}
 
 	if (*commit && !extent_committed_get(extent)) {
-		if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent,
+		if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, 0,
-		    0, extent_size_get(extent))) {
+		    extent_size_get(extent), true)) {
 			extent_record(tsdn, arena, r_extent_hooks,
-			    &arena->extents_retained, extent);
+			    &arena->extents_retained, extent, true);
-			return NULL;
+			goto label_err;
 		}
 		extent_zeroed_set(extent, true);
 	}
 
+	/*
+	 * Increment extent_grow_next if doing so wouldn't exceed the legal
+	 * range.
+	 */
+	if (arena->extent_grow_next + egn_skip + 1 < NPSIZES) {
+		arena->extent_grow_next += egn_skip + 1;
+	} else {
+		arena->extent_grow_next = NPSIZES - 1;
+	}
+	/* All opportunities for failure are past. */
+	malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+
 	if (config_prof) {
 		/* Adjust gdump stats now that extent is final size. */
 		extent_gdump_add(tsdn, extent);
@@ -1150,45 +1174,38 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
 			memset(addr, 0, size);
 		}
 	}
-	/*
+
-	 * Increment extent_grow_next, but take care to do so atomically and
-	 * bail out if the increment would exceed the legal range.
-	 */
-	pszind_t egn = atomic_load_u(&arena->extent_grow_next, ATOMIC_RELAXED);
-	while (true) {
-		if (egn + 1 == NPSIZES) {
-			break;
-		}
-		assert(egn + 1 < NPSIZES);
-		if (atomic_compare_exchange_weak_u(&arena->extent_grow_next,
-		    &egn, egn + 1, ATOMIC_RELAXED, ATOMIC_RELAXED)) {
-			break;
-		}
-	}
 	return extent;
+label_err:
+	malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+	return NULL;
 }
 
 static extent_t *
 extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad,
     size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
-	extent_t *extent;
-
 	assert(size != 0);
 	assert(alignment != 0);
 
-	extent = extent_recycle(tsdn, arena, r_extent_hooks,
+	malloc_mutex_lock(tsdn, &arena->extent_grow_mtx);
+
+	extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks,
 	    &arena->extents_retained, new_addr, size, pad, alignment, slab,
-	    szind, zero, commit);
+	    szind, zero, commit, true);
 	if (extent != NULL) {
+		malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
 		if (config_prof) {
 			extent_gdump_add(tsdn, extent);
 		}
+	} else if (opt_retain && new_addr == NULL) {
+		extent = extent_grow_retained(tsdn, arena, r_extent_hooks, size,
+		    pad, alignment, slab, szind, zero, commit);
+		/* extent_grow_retained() always releases extent_grow_mtx. */
+	} else {
+		malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
 	}
-	if (opt_retain && extent == NULL) {
+	malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx);
-		extent = extent_grow_retained(tsdn, arena, r_extent_hooks,
-		    new_addr, size, pad, alignment, slab, szind, zero, commit);
-	}
 
 	return extent;
 }
@@ -1222,7 +1239,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
 	}
 	if (extent_register(tsdn, extent)) {
 		extents_leak(tsdn, arena, r_extent_hooks,
-		    &arena->extents_retained, extent);
+		    &arena->extents_retained, extent, false);
 		return NULL;
 	}
 
@@ -1270,7 +1287,8 @@ extent_can_coalesce(arena_t *arena, extents_t *extents, const extent_t *inner,
 
 static bool
 extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
-    extents_t *extents, extent_t *inner, extent_t *outer, bool forward) {
+    extents_t *extents, extent_t *inner, extent_t *outer, bool forward,
+    bool growing_retained) {
 	assert(extent_can_coalesce(arena, extents, inner, outer));
 
 	if (forward && extents->delay_coalesce) {
@@ -1285,8 +1303,8 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 	    extents->delay_coalesce);
 
 	malloc_mutex_unlock(tsdn, &extents->mtx);
-	bool err = extent_merge_wrapper(tsdn, arena, r_extent_hooks,
+	bool err = extent_merge_impl(tsdn, arena, r_extent_hooks,
-	    forward ? inner : outer, forward ? outer : inner);
+	    forward ? inner : outer, forward ? outer : inner, growing_retained);
 	malloc_mutex_lock(tsdn, &extents->mtx);
 
 	if (err) {
@@ -1303,7 +1321,7 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 static extent_t *
 extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents,
-    extent_t *extent, bool *coalesced) {
+    extent_t *extent, bool *coalesced, bool growing_retained) {
 	/*
 	 * Continue attempting to coalesce until failure, to protect against
 	 * races with other threads that are thwarted by this one.
@@ -1327,7 +1345,8 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
 			extent_unlock(tsdn, next);
 
 			if (can_coalesce && !extent_coalesce(tsdn, arena,
-			    r_extent_hooks, extents, extent, next, true)) {
+			    r_extent_hooks, extents, extent, next, true,
+			    growing_retained)) {
 				if (extents->delay_coalesce) {
 					/* Do minimal coalescing. */
 					*coalesced = true;
@@ -1346,7 +1365,8 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
 			extent_unlock(tsdn, prev);
 
 			if (can_coalesce && !extent_coalesce(tsdn, arena,
-			    r_extent_hooks, extents, extent, prev, false)) {
+			    r_extent_hooks, extents, extent, prev, false, 
+			    growing_retained)) {
 				extent = prev;
 				if (extents->delay_coalesce) {
 					/* Do minimal coalescing. */
@@ -1366,7 +1386,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
 
 static void
 extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
-    extents_t *extents, extent_t *extent) {
+    extents_t *extents, extent_t *extent, bool growing_retained) {
 	rtree_ctx_t rtree_ctx_fallback;
 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
 
@@ -1388,7 +1408,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 
 	if (!extents->delay_coalesce) {
 		extent = extent_try_coalesce(tsdn, arena, r_extent_hooks,
-		    rtree_ctx, extents, extent, NULL);
+		    rtree_ctx, extents, extent, NULL, growing_retained);
 	}
 
 	extent_deactivate_locked(tsdn, arena, extents, extent, false);
@@ -1405,7 +1425,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
 
 	if (extent_register(tsdn, extent)) {
 		extents_leak(tsdn, arena, &extent_hooks,
-		    &arena->extents_retained, extent);
+		    &arena->extents_retained, extent, false);
 		return;
 	}
 	extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent);
@@ -1503,7 +1523,7 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
 	}
 
 	extent_record(tsdn, arena, r_extent_hooks, &arena->extents_retained,
-	    extent);
+	    extent, false);
 }
 
 static void
@@ -1558,12 +1578,12 @@ extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
 	    length);
 }
 
-bool
+static bool
-extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
+extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
-    size_t length) {
+    size_t length, bool growing_retained) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
+	    WITNESS_RANK_CORE, growing_retained ? 1 : 0);
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
 	bool err = ((*r_extent_hooks)->commit == NULL ||
@@ -1573,6 +1593,14 @@ extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
 	return err;
 }
 
+bool
+extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length) {
+	return extent_commit_impl(tsdn, arena, r_extent_hooks, extent, offset,
+	    length, false);
+}
+
 static bool
 extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
     size_t offset, size_t length, unsigned arena_ind) {
@@ -1614,12 +1642,12 @@ extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
 }
 #endif
 
-bool
+static bool
-extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
+extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
-    size_t length) {
+    size_t length, bool growing_retained) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
+	    WITNESS_RANK_CORE, growing_retained ? 1 : 0);
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
 	return ((*r_extent_hooks)->purge_lazy == NULL ||
@@ -1628,6 +1656,14 @@ extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
 	    arena_ind_get(arena)));
 }
 
+bool
+extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length) {
+	return extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, extent,
+	    offset, length, false);
+}
+
 #ifdef PAGES_CAN_PURGE_FORCED
 static bool
 extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
@@ -1643,12 +1679,12 @@ extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
 }
 #endif
 
-bool
+static bool
-extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
+extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
-    size_t length) {
+    size_t length, bool growing_retained) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
+	    WITNESS_RANK_CORE, growing_retained ? 1 : 0);
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
 	return ((*r_extent_hooks)->purge_forced == NULL ||
@@ -1657,6 +1693,14 @@ extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
 	    arena_ind_get(arena)));
 }
 
+bool
+extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length) {
+	return extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent,
+	    offset, length, false);
+}
+
 #ifdef JEMALLOC_MAPS_COALESCE
 static bool
 extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
@@ -1667,13 +1711,14 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
 }
 #endif
 
-extent_t *
+static extent_t *
-extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
+extent_split_impl(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
-    szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) {
+    szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
+    bool growing_retained) {
 	assert(extent_size_get(extent) == size_a + size_b);
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
+	    WITNESS_RANK_CORE, growing_retained ? 1 : 0);
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
 
@@ -1741,6 +1786,14 @@ label_error_a:
 	return NULL;
 }
 
+extent_t *
+extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
+    szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) {
+	return extent_split_impl(tsdn, arena, r_extent_hooks, extent, size_a,
+	    szind_a, slab_a, size_b, szind_b, slab_b, false);
+}
+
 static bool
 extent_merge_default_impl(void *addr_a, void *addr_b) {
 	if (!maps_coalesce) {
@@ -1763,11 +1816,12 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
 }
 #endif
 
-bool
+static bool
-extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
+extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) {
+    extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b,
+    bool growing_retained) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
+	    WITNESS_RANK_CORE, growing_retained ? 1 : 0);
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
 
@@ -1832,6 +1886,12 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
 	return false;
 }
 
+bool
+extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) {
+	return extent_merge_impl(tsdn, arena, r_extent_hooks, a, b, false);
+}
+
 bool
 extent_boot(void) {
 	if (rtree_new(&extents_rtree, true)) {