Further specialize arena_[s]dalloc() tcache fast path.

Use tsd_rtree_ctx() rather than tsdn_rtree_ctx() when tcache is
non-NULL, in order to avoid an extra branch (and potentially extra stack
space) in the fast path.

include/jemalloc/internal/arena_inlines_b.h

@@ -14,7 +14,9 @@ void	*arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
 arena_t *arena_aalloc(tsdn_t *tsdn, const void *ptr);
 size_t arena_salloc(tsdn_t *tsdn, const void *ptr);
 size_t arena_vsalloc(tsdn_t *tsdn, const void *ptr);
+void arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr);
 void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
+void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size);
 void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
     bool slow_path);
 #endif
@@ -162,9 +164,8 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
 	return index2size(szind);
 }
 
-JEMALLOC_ALWAYS_INLINE void
+JEMALLOC_INLINE void
-arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
+arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
 	assert(ptr != NULL);
 
 	rtree_ctx_t rtree_ctx_fallback;
@@ -179,25 +180,55 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
 		extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
 		    rtree_ctx, (uintptr_t)ptr, true);
 		assert(szind == extent_szind_get(extent));
+		assert(szind < NSIZES);
+		assert(slab == extent_slab_get(extent));
+	}
+
+	if (likely(slab)) {
+		/* Small allocation. */
+		arena_dalloc_small(tsdn, ptr);
+	} else {
+		extent_t *extent = iealloc(tsdn, ptr);
+		large_dalloc(tsdn, extent);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+	assert(ptr != NULL);
+
+	if (unlikely(tcache == NULL)) {
+		arena_dalloc_no_tcache(tsdn, ptr);
+		return;
+	}
+
+	rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
+	szind_t szind;
+	bool slab;
+	rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
+	    true, &szind, &slab);
+
+	if (config_debug) {
+		extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
+		    rtree_ctx, (uintptr_t)ptr, true);
+		assert(szind == extent_szind_get(extent));
+		assert(szind < NSIZES);
 		assert(slab == extent_slab_get(extent));
 	}
 
 	if (likely(slab)) {
 		/* Small allocation. */
-		if (likely(tcache != NULL)) {
 		tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
 		    slow_path);
 	} else {
-			arena_dalloc_small(tsdn, ptr);
+		if (szind < nhbins) {
-		}
-	} else {
-		if (likely(tcache != NULL) && szind < nhbins) {
 			if (config_prof && unlikely(szind < NBINS)) {
 				arena_dalloc_promoted(tsdn, ptr, tcache,
 				    slow_path);
 			} else {
-				tcache_dalloc_large(tsdn_tsd(tsdn), tcache,
+				tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
-				    ptr, szind, slow_path);
+				    szind, slow_path);
 			}
 		} else {
 			extent_t *extent = iealloc(tsdn, ptr);
@@ -206,11 +237,10 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
 	}
 }
 
-JEMALLOC_ALWAYS_INLINE void
+JEMALLOC_INLINE void
-arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
-    bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
 	assert(ptr != NULL);
+	assert(size <= LARGE_MAXCLASS);
 
 	szind_t szind;
 	bool slab;
@@ -244,20 +274,65 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
 
 	if (likely(slab)) {
 		/* Small allocation. */
-		if (likely(tcache != NULL)) {
+		arena_dalloc_small(tsdn, ptr);
+	} else {
+		extent_t *extent = iealloc(tsdn, ptr);
+		large_dalloc(tsdn, extent);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+    bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+	assert(ptr != NULL);
+	assert(size <= LARGE_MAXCLASS);
+
+	if (unlikely(tcache == NULL)) {
+		arena_sdalloc_no_tcache(tsdn, ptr, size);
+		return;
+	}
+
+	szind_t szind;
+	bool slab;
+	if (!config_prof || !opt_prof) {
+		/*
+		 * There is no risk of being confused by a promoted sampled
+		 * object, so base szind and slab on the given size.
+		 */
+		szind = size2index(size);
+		slab = (szind < NBINS);
+	}
+
+	if ((config_prof && opt_prof) || config_debug) {
+		rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
+
+		rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
+		    (uintptr_t)ptr, true, &szind, &slab);
+
+		assert(szind == size2index(size));
+		assert((config_prof && opt_prof) || slab == (szind < NBINS));
+
+		if (config_debug) {
+			extent_t *extent = rtree_extent_read(tsdn,
+			    &extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
+			assert(szind == extent_szind_get(extent));
+			assert(slab == extent_slab_get(extent));
+		}
+	}
+
+	if (likely(slab)) {
+		/* Small allocation. */
 		tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
 		    slow_path);
 	} else {
-			arena_dalloc_small(tsdn, ptr);
+		if (szind < nhbins) {
-		}
-	} else {
-		if (likely(tcache != NULL) && szind < nhbins) {
 			if (config_prof && unlikely(szind < NBINS)) {
 				arena_dalloc_promoted(tsdn, ptr, tcache,
 				    slow_path);
 			} else {
-				tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
+				tcache_dalloc_large(tsdn_tsd(tsdn),
-				    szind, slow_path);
+				    tcache, ptr, szind, slow_path);
 			}
 		} else {
 			extent_t *extent = iealloc(tsdn, ptr);

include/jemalloc/internal/private_symbols.txt

@@ -13,6 +13,7 @@ arena_cleanup
 arena_dalloc
 arena_dalloc_bin_junked_locked
 arena_dalloc_junk_small
+arena_dalloc_no_tcache
 arena_dalloc_promoted
 arena_dalloc_small
 arena_decay
@@ -69,6 +70,7 @@ arena_ralloc_no_move
 arena_reset
 arena_salloc
 arena_sdalloc
+arena_sdalloc_no_tcache
 arena_set
 arena_slab_regind
 arena_stats_init
@@ -528,6 +530,7 @@ tsd_nominal
 tsd_prof_tdata_get
 tsd_prof_tdata_set
 tsd_prof_tdatap_get
+tsd_rtree_ctx
 tsd_rtree_ctxp_get
 tsd_rtree_leaf_elm_witnessesp_get
 tsd_set

include/jemalloc/internal/tsd_inlines.h

@@ -17,6 +17,7 @@ MALLOC_TSD
 tsdn_t *tsdn_fetch(void);
 bool tsdn_null(const tsdn_t *tsdn);
 tsd_t *tsdn_tsd(tsdn_t *tsdn);
+rtree_ctx_t *tsd_rtree_ctx(tsd_t *tsd);
 rtree_ctx_t *tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback);
 #endif
 
@@ -108,6 +109,11 @@ tsdn_tsd(tsdn_t *tsdn) {
 	return &tsdn->tsd;
 }
 
+JEMALLOC_ALWAYS_INLINE rtree_ctx_t *
+tsd_rtree_ctx(tsd_t *tsd) {
+	return tsd_rtree_ctxp_get(tsd);
+}
+
 JEMALLOC_ALWAYS_INLINE rtree_ctx_t *
 tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback) {
 	/*
@@ -119,7 +125,7 @@ tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback) {
 		memcpy(fallback, &rtree_ctx, sizeof(rtree_ctx_t));
 		return fallback;
 	}
-	return tsd_rtree_ctxp_get(tsdn_tsd(tsdn));
+	return tsd_rtree_ctx(tsdn_tsd(tsdn));
 }
 #endif