Integrate auto tcache into TSD.

The embedded tcache is initialized upon tsd initialization.  The avail arrays
for the tbins will be allocated / deallocated accordingly during init / cleanup.

With this change, the pointer to the auto tcache will always be available, as
long as we have access to the TSD.  tcache_available() (called in tcache_get())
is provided to check if we should use tcache.

include/jemalloc/internal/arena_inlines_a.h

@@ -57,12 +57,10 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
 
 		/* Set new arena/tcache associations. */
 		arena_migrate(tsd, oldind, newind);
-		if (config_tcache) {
-			tcache_t *tcache = tsd_tcache_get(tsd);
-			if (tcache) {
-				tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
-				    newarena);
-			}
+		tcache_t *tcache = tcache_get(tsd);
+		if (config_tcache && tcache) {
+			tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
+			    newarena);
 		}
 	}
 }

include/jemalloc/internal/jemalloc_internal.h.in

@@ -488,23 +488,24 @@ extern size_t const	index2size_tab[NSIZES];
  */
 extern uint8_t const	size2index_tab[];
 
-void	*a0malloc(size_t size);
-void	a0dalloc(void *ptr);
-void	*bootstrap_malloc(size_t size);
-void	*bootstrap_calloc(size_t num, size_t size);
-void	bootstrap_free(void *ptr);
-void	arena_set(unsigned ind, arena_t *arena);
-unsigned	narenas_total_get(void);
-arena_t	*arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
-arena_tdata_t	*arena_tdata_get_hard(tsd_t *tsd, unsigned ind);
-arena_t	*arena_choose_hard(tsd_t *tsd, bool internal);
-void	arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind);
-void	iarena_cleanup(tsd_t *tsd);
-void	arena_cleanup(tsd_t *tsd);
-void	arenas_tdata_cleanup(tsd_t *tsd);
-void	jemalloc_prefork(void);
-void	jemalloc_postfork_parent(void);
-void	jemalloc_postfork_child(void);
+void *a0malloc(size_t size);
+void a0dalloc(void *ptr);
+void *bootstrap_malloc(size_t size);
+void *bootstrap_calloc(size_t num, size_t size);
+void bootstrap_free(void *ptr);
+void arena_set(unsigned ind, arena_t *arena);
+unsigned narenas_total_get(void);
+arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
+arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind);
+arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
+void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind);
+void iarena_cleanup(tsd_t *tsd);
+void arena_cleanup(tsd_t *tsd);
+void arenas_tdata_cleanup(tsd_t *tsd);
+void jemalloc_prefork(void);
+void jemalloc_postfork_parent(void);
+void jemalloc_postfork_child(void);
+bool malloc_initialized(void);
 
 #include "jemalloc/internal/nstime_externs.h"
 #include "jemalloc/internal/ckh_externs.h"
@@ -559,6 +560,8 @@ arena_tdata_t	*arena_tdata_get(tsd_t *tsd, unsigned ind,
     bool refresh_if_missing);
 arena_t	*arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing);
 ticker_t	*decay_ticker_get(tsd_t *tsd, unsigned ind);
+bool	tcache_available(tsd_t *tsd);
+tcache_t	*tcache_get(tsd_t *tsd);
 malloc_cpuid_t	malloc_getcpu(void);
 unsigned	percpu_arena_choose(void);
 unsigned	percpu_arena_ind_limit(void);
@@ -929,6 +932,38 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) {
 	}
 	return &tdata->decay_ticker;
 }
+
+JEMALLOC_ALWAYS_INLINE bool
+tcache_available(tsd_t *tsd) {
+	cassert(config_tcache);
+
+	/*
+	 * Thread specific auto tcache might be unavailable if: 1) during tcache
+	 * initialization, or 2) disabled through thread.tcache.enabled mallctl
+	 * or config options.  This check covers all cases.
+	 */
+	if (likely(tsd_tcache_enabled_get(tsd) == tcache_enabled_true)) {
+		/* Associated arena == null implies tcache init in progress. */
+		if (tsd_tcachep_get(tsd)->arena != NULL) {
+			assert(tsd_tcachep_get(tsd)->tbins[0].avail != NULL);
+		}
+		return true;
+	}
+
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE tcache_t *
+tcache_get(tsd_t *tsd) {
+	if (!config_tcache) {
+		return NULL;
+	}
+	if (!tcache_available(tsd)) {
+		return NULL;
+	}
+
+	return tsd_tcachep_get(tsd);
+}
 #endif
 
 #include "jemalloc/internal/rtree_inlines.h"
@@ -959,9 +994,24 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
 	ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
 	if (unlikely(ret == NULL)) {
 		ret = arena_choose_hard(tsd, internal);
+		assert(ret);
+		if (config_tcache && tcache_available(tsd)) {
+			tcache_t *tcache = tcache_get(tsd);
+			if (tcache->arena != NULL) {
+				/* See comments in tcache_data_init().*/
+				assert(tcache->arena ==
+				    arena_get(tsd_tsdn(tsd), 0, false));
+				if (tcache->arena != ret) {
+					tcache_arena_reassociate(tsd_tsdn(tsd),
+					    tcache, ret);
+				}
+			} else {
+				tcache_arena_associate(tsd_tsdn(tsd), tcache,
+				    ret);
+			}
+		}
 	}
 
-	assert(ret != NULL);
 	/*
 	 * Note that for percpu arena, if the current arena is outside of the
 	 * auto percpu arena range, (i.e. thread is assigned to a manually
@@ -1069,8 +1119,8 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
 
 JEMALLOC_ALWAYS_INLINE void *
 ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) {
-	return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd, true),
-	    false, NULL, slow_path);
+	return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd), false,
+	    NULL, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -1102,7 +1152,7 @@ ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
 JEMALLOC_ALWAYS_INLINE void *
 ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) {
 	return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero,
-	    tcache_get(tsd, true), false, NULL);
+	    tcache_get(tsd), false, NULL);
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
@@ -1127,7 +1177,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_internal,
 
 JEMALLOC_ALWAYS_INLINE void
 idalloc(tsd_t *tsd, void *ptr) {
-	idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd, false), false, true);
+	idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd), false, true);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -1199,7 +1249,7 @@ JEMALLOC_ALWAYS_INLINE void *
 iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
     bool zero) {
 	return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero,
-	    tcache_get(tsd, true), NULL);
+	    tcache_get(tsd), NULL);
 }
 
 JEMALLOC_ALWAYS_INLINE bool

include/jemalloc/internal/private_symbols.txt

@@ -274,6 +274,7 @@ lg_floor
 lg_prof_sample
 malloc_cprintf
 malloc_getcpu
+malloc_initialized
 malloc_mutex_prof_data_reset
 malloc_mutex_assert_not_owner
 malloc_mutex_assert_owner
@@ -293,7 +294,6 @@ malloc_tsd_boot1
 malloc_tsd_cleanup_register
 malloc_tsd_dalloc
 malloc_tsd_malloc
-malloc_tsd_no_cleanup
 malloc_vcprintf
 malloc_vsnprintf
 malloc_write
@@ -475,22 +475,23 @@ tcache_alloc_easy
 tcache_alloc_large
 tcache_alloc_small
 tcache_alloc_small_hard
+tcache_arena_associate
 tcache_arena_reassociate
 tcache_bin_flush_large
 tcache_bin_flush_small
 tcache_bin_info
 tcache_boot
 tcache_cleanup
-tcache_create
+tcache_create_explicit
 tcache_dalloc_large
 tcache_dalloc_small
+tcache_data_init
 tcache_enabled_get
 tcache_enabled_set
 tcache_event
 tcache_event_hard
 tcache_flush
 tcache_get
-tcache_get_hard
 tcache_maxclass
 tcache_prefork
 tcache_postfork_child
@@ -521,7 +522,6 @@ tsd_booted
 tsd_booted_get
 tsd_cleanup
 tsd_cleanup_wrapper
-tsd_data_init
 tsd_fetch
 tsd_fetch_impl
 tsd_get

include/jemalloc/internal/rtree_inlines.h

@@ -321,13 +321,18 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	assert(!dependent || !init_missing);
 
 	uintptr_t leafkey = rtree_leafkey(key);
+	assert(leafkey != RTREE_LEAFKEY_INVALID);
+
 #define RTREE_CACHE_CHECK(i) do {					\
 	if (likely(rtree_ctx->cache[i].leafkey == leafkey)) {		\
 		rtree_leaf_elm_t *leaf = rtree_ctx->cache[i].leaf;	\
 		assert(leaf != NULL);					\
 		if (i > 0) {						\
 			/* Bubble up by one. */				\
-			rtree_ctx->cache[i] = rtree_ctx->cache[i - 1];	\
+			rtree_ctx->cache[i].leafkey =			\
+				rtree_ctx->cache[i - 1].leafkey;	\
+			rtree_ctx->cache[i].leaf =			\
+				rtree_ctx->cache[i - 1].leaf;		\
 			rtree_ctx->cache[i - 1].leafkey = leafkey;	\
 			rtree_ctx->cache[i - 1].leaf = leaf;		\
 		}							\

include/jemalloc/internal/tcache_externs.h

@@ -35,16 +35,19 @@ void	tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
     unsigned rem, tcache_t *tcache);
 void	tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
     arena_t *arena);
-tcache_t *tcache_get_hard(tsd_t *tsd);
-tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena);
+tcache_t *tcache_create_explicit(tsd_t *tsd);
 void	tcache_cleanup(tsd_t *tsd);
 void	tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
 bool	tcaches_create(tsd_t *tsd, unsigned *r_ind);
 void	tcaches_flush(tsd_t *tsd, unsigned ind);
 void	tcaches_destroy(tsd_t *tsd, unsigned ind);
 bool	tcache_boot(tsdn_t *tsdn);
+void tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
 void tcache_prefork(tsdn_t *tsdn);
 void tcache_postfork_parent(tsdn_t *tsdn);
 void tcache_postfork_child(tsdn_t *tsdn);
+void tcache_flush(void);
+bool tsd_tcache_data_init(tsd_t *tsd);
+bool tsd_tcache_enabled_data_init(tsd_t *tsd);
 
 #endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */

include/jemalloc/internal/tcache_inlines.h

@@ -4,9 +4,9 @@
 #ifndef JEMALLOC_ENABLE_INLINE
 void	tcache_event(tsd_t *tsd, tcache_t *tcache);
 void	tcache_flush(void);
-bool	tcache_enabled_get(void);
-tcache_t *tcache_get(tsd_t *tsd, bool create);
-void	tcache_enabled_set(bool enabled);
+bool	tcache_enabled_get(tsd_t *tsd);
+tcache_t *tcache_get(tsd_t *tsd);
+void	tcache_enabled_set(tsd_t *tsd, bool enabled);
 void	*tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
 void	*tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
     size_t size, szind_t ind, bool zero, bool slow_path);
@@ -20,68 +20,32 @@ tcache_t	*tcaches_get(tsd_t *tsd, unsigned ind);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
-JEMALLOC_INLINE void
-tcache_flush(void) {
-	tsd_t *tsd;
-
-	cassert(config_tcache);
-
-	tsd = tsd_fetch();
-	tcache_cleanup(tsd);
-}
-
 JEMALLOC_INLINE bool
-tcache_enabled_get(void) {
-	tsd_t *tsd;
+tcache_enabled_get(tsd_t *tsd) {
 	tcache_enabled_t tcache_enabled;
 
 	cassert(config_tcache);
 
-	tsd = tsd_fetch();
 	tcache_enabled = tsd_tcache_enabled_get(tsd);
-	if (tcache_enabled == tcache_enabled_default) {
-		tcache_enabled = (tcache_enabled_t)opt_tcache;
-		tsd_tcache_enabled_set(tsd, tcache_enabled);
-	}
+	assert(tcache_enabled != tcache_enabled_default);
 
 	return (bool)tcache_enabled;
 }
 
 JEMALLOC_INLINE void
-tcache_enabled_set(bool enabled) {
-	tsd_t *tsd;
-	tcache_enabled_t tcache_enabled;
-
+tcache_enabled_set(tsd_t *tsd, bool enabled) {
 	cassert(config_tcache);
 
-	tsd = tsd_fetch();
+	tcache_enabled_t old = tsd_tcache_enabled_get(tsd);
 
-	tcache_enabled = (tcache_enabled_t)enabled;
-	tsd_tcache_enabled_set(tsd, tcache_enabled);
-
-	if (!enabled) {
+	if ((old != tcache_enabled_true) && enabled) {
+		tsd_tcache_data_init(tsd);
+	} else if ((old == tcache_enabled_true) && !enabled) {
 		tcache_cleanup(tsd);
 	}
-}
-
-JEMALLOC_ALWAYS_INLINE tcache_t *
-tcache_get(tsd_t *tsd, bool create) {
-	tcache_t *tcache;
-
-	if (!config_tcache) {
-		return NULL;
-	}
-
-	tcache = tsd_tcache_get(tsd);
-	if (!create) {
-		return tcache;
-	}
-	if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
-		tcache = tcache_get_hard(tsd);
-		tsd_tcache_set(tsd, tcache);
-	}
-
-	return tcache;
+	/* Commit the state last.  Above calls check current state. */
+	tcache_enabled_t tcache_enabled = (tcache_enabled_t)enabled;
+	tsd_tcache_enabled_set(tsd, tcache_enabled);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -300,8 +264,7 @@ JEMALLOC_ALWAYS_INLINE tcache_t *
 tcaches_get(tsd_t *tsd, unsigned ind) {
 	tcaches_t *elm = &tcaches[ind];
 	if (unlikely(elm->tcache == NULL)) {
-		elm->tcache = tcache_create(tsd_tsdn(tsd), arena_choose(tsd,
-		    NULL));
+		elm->tcache = tcache_create_explicit(tsd);
 	}
 	return elm->tcache;
 }

include/jemalloc/internal/tcache_structs.h

@@ -36,13 +36,17 @@ struct tcache_s {
 	ticker_t	gc_ticker;	/* Drives incremental GC. */
 	szind_t		next_gc_bin;	/* Next bin to GC. */
 	arena_t		*arena;		/* Associated arena. */
-	tcache_bin_t	tbins[1];	/* Dynamically sized. */
 	/*
 	 * The pointer stacks associated with tbins follow as a contiguous
 	 * array.  During tcache initialization, the avail pointer in each
 	 * element of tbins is initialized to point to the proper offset within
 	 * this array.
 	 */
+#ifdef JEMALLOC_TCACHE
+	tcache_bin_t	tbins[NSIZES];
+#else
+	tcache_bin_t	tbins[0];
+#endif
 };
 
 /* Linkage for list of available (previously used) explicit tcache IDs. */

include/jemalloc/internal/tcache_types.h

@@ -47,4 +47,7 @@ typedef struct tcaches_s tcaches_t;
 #define TCACHE_GC_INCR							\
     ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1))
 
+/* Used in TSD static initializer only. Real init in tcache_data_init(). */
+#define TCACHE_ZERO_INITIALIZER {{NULL}}
+
 #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */

include/jemalloc/internal/tsd_externs.h

@@ -3,7 +3,6 @@
 
 void	*malloc_tsd_malloc(size_t size);
 void	malloc_tsd_dalloc(void *wrapper);
-void	malloc_tsd_no_cleanup(void *arg);
 void	malloc_tsd_cleanup_register(bool (*f)(void));
 tsd_t	*malloc_tsd_boot0(void);
 void	malloc_tsd_boot1(void);
@@ -13,7 +12,7 @@ void	*tsd_init_check_recursion(tsd_init_head_t *head,
     tsd_init_block_t *block);
 void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
 #endif
-void	tsd_cleanup(void *arg);
-bool	tsd_data_init(void *arg);
+bool tsd_data_init(void *arg);
+void tsd_cleanup(void *arg);
 
 #endif /* JEMALLOC_INTERNAL_TSD_EXTERNS_H */

include/jemalloc/internal/tsd_structs.h

@@ -16,7 +16,7 @@ struct tsd_init_head_s {
 
 #define MALLOC_TSD							\
 /*  O(name,			type,		[gs]et,	init,	cleanup) */ \
-    O(tcache,			tcache_t *,	yes,	no,	yes)	\
+    O(tcache,			tcache_t,	yes,	no,	yes)	\
     O(thread_allocated,		uint64_t,	yes,	no,	no)	\
     O(thread_deallocated,	uint64_t,	yes,	no,	no)	\
     O(prof_tdata,		prof_tdata_t *,	yes,	no,	yes)	\
@@ -26,7 +26,7 @@ struct tsd_init_head_s {
     O(narenas_tdata,		unsigned,	yes,	no,	no)	\
     O(arenas_tdata_bypass,	bool,		no,	no,	no)	\
     O(tcache_enabled,		tcache_enabled_t,			\
-						yes,	no,	no)	\
+						yes,	yes,	no)	\
     O(rtree_ctx,		rtree_ctx_t,	no,	yes,	no)	\
     O(witnesses,		witness_list_t,	no,	no,	yes)	\
     O(rtree_leaf_elm_witnesses,	rtree_leaf_elm_witness_tsd_t,		\
@@ -35,7 +35,7 @@ struct tsd_init_head_s {
 
 #define TSD_INITIALIZER {						\
     tsd_state_uninitialized,						\
-    NULL,								\
+    TCACHE_ZERO_INITIALIZER,						\
     0,									\
     0,									\
     NULL,								\
@@ -69,6 +69,7 @@ struct tsdn_s {
 };
 
 static const tsd_t tsd_initializer = TSD_INITIALIZER;
+UNUSED static const void *malloc_tsd_no_cleanup = (void *)0;
 
 malloc_tsd_types(, tsd_t)
 

include/jemalloc/internal/tsd_types.h

@@ -357,8 +357,10 @@ a_name##tsd_boot1(void) {						\
 		    " TSD for "#a_name"\n");				\
 		abort();						\
 	}								\
-	memcpy(wrapper, &a_name##tsd_boot_wrapper,			\
-	    sizeof(a_name##tsd_wrapper_t));				\
+	a_name##tsd_boot_wrapper.initialized = false;			\
+	a_cleanup(&a_name##tsd_boot_wrapper.val);			\
+	wrapper->initialized = false;					\
+	wrapper->val = a_initializer;					\
 	a_name##tsd_wrapper_set(wrapper);				\
 }									\
 a_attr bool								\
@@ -487,8 +489,10 @@ a_name##tsd_boot1(void) {						\
 		    " TSD for "#a_name"\n");				\
 		abort();						\
 	}								\
-	memcpy(wrapper, &a_name##tsd_boot_wrapper,			\
-	    sizeof(a_name##tsd_wrapper_t));				\
+	a_name##tsd_boot_wrapper.initialized = false;			\
+	a_cleanup(&a_name##tsd_boot_wrapper.val);			\
+	wrapper->initialized = false;					\
+	wrapper->val = a_initializer;					\
 	a_name##tsd_wrapper_set(wrapper);				\
 }									\
 a_attr bool								\