Tcache: split up fast and slow path data.

include/jemalloc/internal/arena_structs.h

@@ -53,7 +53,7 @@ struct arena_s {
 	 *
 	 * Synchronization: tcache_ql_mtx.
 	 */
-	ql_head(tcache_t)			tcache_ql;
+	ql_head(tcache_slow_t)			tcache_ql;
 	ql_head(cache_bin_array_descriptor_t)	cache_bin_array_descriptor_ql;
 	malloc_mutex_t				tcache_ql_mtx;
 

include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -129,7 +129,7 @@ tcache_available(tsd_t *tsd) {
 	 */
 	if (likely(tsd_tcache_enabled_get(tsd))) {
 		/* Associated arena == NULL implies tcache init in progress. */
-		assert(tsd_tcachep_get(tsd)->arena == NULL ||
+		assert(tsd_tcache_slowp_get(tsd)->arena == NULL ||
 		    !cache_bin_still_zero_initialized(
 		    tcache_small_bin_get(tsd_tcachep_get(tsd), 0)));
 		return true;
@@ -147,6 +147,15 @@ tcache_get(tsd_t *tsd) {
 	return tsd_tcachep_get(tsd);
 }
 
+JEMALLOC_ALWAYS_INLINE tcache_slow_t *
+tcache_slow_get(tsd_t *tsd) {
+	if (!tcache_available(tsd)) {
+		return NULL;
+	}
+
+	return tsd_tcache_slowp_get(tsd);
+}
+
 static inline void
 pre_reentrancy(tsd_t *tsd, arena_t *arena) {
 	/* arena is the current context.  Reentry from a0 is not allowed. */

include/jemalloc/internal/jemalloc_internal_inlines_b.h

@@ -19,8 +19,10 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
 		arena_migrate(tsd, oldind, newind);
 		tcache_t *tcache = tcache_get(tsd);
 		if (tcache != NULL) {
-			tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
-			    newarena);
+			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
+			tcache_t *tcache = tsd_tcachep_get(tsd);
+			tcache_arena_reassociate(tsd_tsdn(tsd), tcache_slow,
+			    tcache, newarena);
 		}
 	}
 }
@@ -45,18 +47,19 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
 		ret = arena_choose_hard(tsd, internal);
 		assert(ret);
 		if (tcache_available(tsd)) {
-			tcache_t *tcache = tcache_get(tsd);
-			if (tcache->arena != NULL) {
+			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
+			tcache_t *tcache = tsd_tcachep_get(tsd);
+			if (tcache_slow->arena != NULL) {
 				/* See comments in tsd_tcache_data_init().*/
-				assert(tcache->arena ==
+				assert(tcache_slow->arena ==
 				    arena_get(tsd_tsdn(tsd), 0, false));
-				if (tcache->arena != ret) {
+				if (tcache_slow->arena != ret) {
 					tcache_arena_reassociate(tsd_tsdn(tsd),
-					    tcache, ret);
+					    tcache_slow, tcache, ret);
 				}
 			} else {
-				tcache_arena_associate(tsd_tsdn(tsd), tcache,
-				    ret);
+				tcache_arena_associate(tsd_tsdn(tsd),
+				    tcache_slow, tcache, ret);
 			}
 		}
 	}

include/jemalloc/internal/tcache_externs.h

@@ -26,15 +26,17 @@ extern cache_bin_info_t *tcache_bin_info;
 extern tcaches_t	*tcaches;
 
 size_t	tcache_salloc(tsdn_t *tsdn, const void *ptr);
-void	tcache_event_hard(tsd_t *tsd, tcache_t *tcache);
+void	tcache_event_hard(tsd_t *tsd, tcache_slow_t *tcache_slow,
+    tcache_t *tcache);
 void	*tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
     cache_bin_t *tbin, szind_t binind, bool *tcache_success);
+
 void	tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
     szind_t binind, unsigned rem);
 void	tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
     szind_t binind, unsigned rem);
-void	tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
-    arena_t *arena);
+void	tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
+    tcache_t *tcache, arena_t *arena);
 tcache_t *tcache_create_explicit(tsd_t *tsd);
 void	tcache_cleanup(tsd_t *tsd);
 void	tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
@@ -42,7 +44,8 @@ bool	tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
 void	tcaches_flush(tsd_t *tsd, unsigned ind);
 void	tcaches_destroy(tsd_t *tsd, unsigned ind);
 bool	tcache_boot(tsdn_t *tsdn, base_t *base);
-void tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
+void tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
+    tcache_t *tcache, arena_t *arena);
 void tcache_prefork(tsdn_t *tsdn);
 void tcache_postfork_parent(tsdn_t *tsdn);
 void tcache_postfork_child(tsdn_t *tsdn);

include/jemalloc/internal/tcache_structs.h

@@ -7,25 +7,19 @@
 #include "jemalloc/internal/ticker.h"
 #include "jemalloc/internal/tsd_types.h"
 
-struct tcache_s {
-	/*
-	 * To minimize our cache-footprint, we put the frequently accessed data
-	 * together at the start of this struct.
-	 */
+/*
+ * The tcache state is split into the slow and hot path data.  Each has a
+ * pointer to the other, and the data always comes in pairs.  The layout of each
+ * of them varies in practice; tcache_slow lives in the TSD for the automatic
+ * tcache, and as part of a dynamic allocation for manual allocations.  Keeping
+ * a pointer to tcache_slow lets us treat these cases uniformly, rather than
+ * splitting up the tcache [de]allocation code into those paths called with the
+ * TSD tcache and those called with a manual tcache.
+ */
 
-	/*
-	 * The pointer stacks associated with bins follow as a contiguous array.
-	 * During tcache initialization, the avail pointer in each element of
-	 * tbins is initialized to point to the proper offset within this array.
-	 */
-	cache_bin_t	bins_small[SC_NBINS];
-
-	/*
-	 * This data is less hot; we can be a little less careful with our
-	 * footprint here.
-	 */
+struct tcache_slow_s {
 	/* Lets us track all the tcaches in an arena. */
-	ql_elm(tcache_t) link;
+	ql_elm(tcache_slow_t) link;
 
 	/*
 	 * The descriptor lets the arena find our cache bins without seeing the
@@ -45,9 +39,23 @@ struct tcache_s {
 	/*
 	 * The start of the allocation containing the dynamic allocation for
 	 * either the cache bins alone, or the cache bin memory as well as this
-	 * tcache_t.
+	 * tcache_slow_t and its associated tcache_t.
 	 */
 	void		*dyn_alloc;
+
+	/* The associated bins. */
+	tcache_t	*tcache;
+};
+
+struct tcache_s {
+	tcache_slow_t	*tcache_slow;
+	/*
+	 * The pointer stacks associated with bins follow as a contiguous array.
+	 * During tcache initialization, the avail pointer in each element of
+	 * tbins is initialized to point to the proper offset within this array.
+	 */
+	cache_bin_t	bins_small[SC_NBINS];
+
 	/*
 	 * We put the cache bins for large size classes at the end of the
 	 * struct, since some of them might not get used.  This might end up

include/jemalloc/internal/tcache_types.h

@@ -3,6 +3,7 @@
 
 #include "jemalloc/internal/sc.h"
 
+typedef struct tcache_slow_s tcache_slow_t;
 typedef struct tcache_s tcache_t;
 typedef struct tcaches_s tcaches_t;
 
@@ -52,6 +53,7 @@ typedef struct tcaches_s tcaches_t;
 
 /* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
 #define TCACHE_ZERO_INITIALIZER {0}
+#define TCACHE_SLOW_ZERO_INITIALIZER {0}
 
 /* Used in TSD static initializer only. Will be initialized to opt_tcache. */
 #define TCACHE_ENABLED_ZERO_INITIALIZER false

include/jemalloc/internal/tsd.h

@@ -76,7 +76,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
     O(arenas_tdata,		arena_tdata_t *,	arena_tdata_t *)\
     O(binshards,		tsd_binshards_t,	tsd_binshards_t)\
     O(tsd_link,			tsd_link_t,		tsd_link_t)	\
-    O(in_hook,			bool,			bool)
+    O(in_hook,			bool,			bool)		\
+    O(tcache_slow,		tcache_slow_t,		tcache_slow_t)
 
 #define TSD_DATA_SLOW_INITIALIZER					\
     /* tcache_enabled */	TCACHE_ENABLED_ZERO_INITIALIZER,	\
@@ -100,7 +101,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
     /* arenas_tdata */		NULL,					\
     /* binshards */		TSD_BINSHARDS_ZERO_INITIALIZER,		\
     /* tsd_link */		{NULL},					\
-    /* in_hook */		false,
+    /* in_hook */		false,					\
+    /* tcache_slow */		TCACHE_SLOW_ZERO_INITIALIZER,
 
 /*  O(name,			type,			nullable type) */
 #define TSD_DATA_FAST							\