Bundle 3 branches on fast path into tsd_state.

Added tsd_state_nominal_slow, which on fast path malloc() incorporates
tcache_enabled check, and on fast path free() bundles both malloc_slow and
tcache_enabled branches.

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -3,6 +3,10 @@
 
 #include "jemalloc/internal/atomic.h"
 
+/* TSD checks this to set thread local slow state accordingly. */
+extern bool	malloc_slow;
+
+/* Run-time options. */
 extern bool	opt_abort;
 extern const char	*opt_junk;
 extern bool	opt_junk_alloc;

include/jemalloc/internal/private_symbols.txt

@@ -289,6 +289,7 @@ malloc_mutex_postfork_parent
 malloc_mutex_prefork
 malloc_mutex_unlock
 malloc_printf
+malloc_slow
 malloc_snprintf
 malloc_strtoumax
 malloc_tsd_boot0
@@ -526,6 +527,7 @@ tsd_cleanup
 tsd_cleanup_wrapper
 tsd_fetch
 tsd_fetch_impl
+tsd_fetch_slow
 tsd_get
 tsd_get_allocates
 tsd_iarena_get
@@ -541,6 +543,7 @@ tsd_narenas_tdatap_get
 tsd_reentrancy_level_get
 tsd_reentrancy_level_set
 tsd_reentrancy_levelp_get
+tsd_slow_update
 tsd_wrapper_get
 tsd_wrapper_set
 tsd_nominal

include/jemalloc/internal/tcache_inlines.h

@@ -40,6 +40,7 @@ tcache_enabled_set(tsd_t *tsd, bool enabled) {
 	}
 	/* Commit the state last.  Above calls check current state. */
 	tsd_tcache_enabled_set(tsd, enabled);
+	tsd_slow_update(tsd);
 }
 
 JEMALLOC_ALWAYS_INLINE void

include/jemalloc/internal/tsd_externs.h

@@ -14,5 +14,7 @@ void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
 #endif
 bool tsd_data_init(void *arg);
 void tsd_cleanup(void *arg);
+tsd_t *tsd_fetch_slow(tsd_t *tsd);
+void tsd_slow_update(tsd_t *tsd);
 
 #endif /* JEMALLOC_INTERNAL_TSD_EXTERNS_H */

include/jemalloc/internal/tsd_inlines.h

@@ -19,12 +19,54 @@ bool tsdn_null(const tsdn_t *tsdn);
 tsd_t *tsdn_tsd(tsdn_t *tsdn);
 rtree_ctx_t *tsd_rtree_ctx(tsd_t *tsd);
 rtree_ctx_t *tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback);
+bool tsd_fast(tsd_t *tsd);
+void tsd_assert_fast(tsd_t *tsd);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
 malloc_tsd_externs(, tsd_t)
 malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
 
+#define MALLOC_TSD_getset_yes(n, t)					\
+JEMALLOC_ALWAYS_INLINE t						\
+tsd_##n##_get(tsd_t *tsd) {						\
+	return *tsd_##n##p_get(tsd);					\
+}									\
+JEMALLOC_ALWAYS_INLINE void						\
+tsd_##n##_set(tsd_t *tsd, t n) {					\
+	assert(tsd->state == tsd_state_nominal ||			\
+	    tsd->state == tsd_state_nominal_slow ||			\
+	    tsd->state == tsd_state_reincarnated);			\
+	tsd->n = n;							\
+}
+#define MALLOC_TSD_getset_no(n, t)
+#define O(n, t, gs, i, c)						\
+JEMALLOC_ALWAYS_INLINE t *						\
+tsd_##n##p_get(tsd_t *tsd) {						\
+	return &tsd->n;							\
+}									\
+									\
+MALLOC_TSD_getset_##gs(n, t)
+MALLOC_TSD
+#undef MALLOC_TSD_getset_yes
+#undef MALLOC_TSD_getset_no
+#undef O
+
+JEMALLOC_ALWAYS_INLINE void
+tsd_assert_fast(tsd_t *tsd) {
+	assert(!malloc_slow && tsd_tcache_enabled_get(tsd));
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+tsd_fast(tsd_t *tsd) {
+	bool fast = (tsd->state == tsd_state_nominal);
+	if (fast) {
+		tsd_assert_fast(tsd);
+	}
+
+	return fast;
+}
+
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_fetch_impl(bool init) {
 	tsd_t *tsd = tsd_get(init);
@@ -35,19 +77,10 @@ tsd_fetch_impl(bool init) {
 	assert(tsd != NULL);
 
 	if (unlikely(tsd->state != tsd_state_nominal)) {
-		if (tsd->state == tsd_state_uninitialized) {
-			tsd->state = tsd_state_nominal;
-			/* Trigger cleanup handler registration. */
-			tsd_set(tsd);
-			tsd_data_init(tsd);
-		} else if (tsd->state == tsd_state_purgatory) {
-			tsd->state = tsd_state_reincarnated;
-			tsd_set(tsd);
-			tsd_data_init(tsd);
-		} else {
-			assert(tsd->state == tsd_state_reincarnated);
-		}
+		return tsd_fetch_slow(tsd);
 	}
+	assert(tsd_fast(tsd));
+	tsd_assert_fast(tsd);
 
 	return tsd;
 }
@@ -64,33 +97,9 @@ tsd_tsdn(tsd_t *tsd) {
 
 JEMALLOC_INLINE bool
 tsd_nominal(tsd_t *tsd) {
-	return (tsd->state == tsd_state_nominal);
+	return (tsd->state <= tsd_state_nominal_max);
 }
 
-#define MALLOC_TSD_getset_yes(n, t)					\
-JEMALLOC_ALWAYS_INLINE t						\
-tsd_##n##_get(tsd_t *tsd) {						\
-	return *tsd_##n##p_get(tsd);					\
-}									\
-JEMALLOC_ALWAYS_INLINE void						\
-tsd_##n##_set(tsd_t *tsd, t n) {					\
-	assert(tsd->state == tsd_state_nominal ||			\
-	    tsd->state == tsd_state_reincarnated);			\
-	tsd->n = n;							\
-}
-#define MALLOC_TSD_getset_no(n, t)
-#define O(n, t, gs, i, c)						\
-JEMALLOC_ALWAYS_INLINE t *						\
-tsd_##n##p_get(tsd_t *tsd) {						\
-	return &tsd->n;							\
-}									\
-									\
-MALLOC_TSD_getset_##gs(n, t)
-MALLOC_TSD
-#undef MALLOC_TSD_getset_yes
-#undef MALLOC_TSD_getset_no
-#undef O
-
 JEMALLOC_ALWAYS_INLINE tsdn_t *
 tsdn_fetch(void) {
 	if (!tsd_booted_get()) {

include/jemalloc/internal/tsd_structs.h

@@ -64,7 +64,7 @@ struct tsd_init_head_s {
     O(iarena,			arena_t *,	yes,	no,	yes)	\
     O(arena,			arena_t *,	yes,	no,	yes)	\
     O(arenas_tdata,		arena_tdata_t *,yes,	no,	yes)	\
-    O(tcache,			tcache_t,	yes,	no,	yes)	\
+    O(tcache,			tcache_t,	no,	no,	yes)	\
     O(witnesses,		witness_list_t,	no,	no,	yes)	\
     O(rtree_leaf_elm_witnesses,	rtree_leaf_elm_witness_tsd_t,		\
 						no,	no,	no)	\

include/jemalloc/internal/tsd_types.h

@@ -20,11 +20,15 @@ typedef struct tsdn_s tsdn_t;
 #define TSDN_NULL	((tsdn_t *)0)
 
 enum {
-	tsd_state_uninitialized = 0,
-	tsd_state_nominal = 1,
+	tsd_state_nominal = 0, /* Common case --> jnz. */
+	tsd_state_nominal_slow = 1, /* Initialized but on slow path. */
+	/* the above 2 nominal states should be lower values. */
+	tsd_state_nominal_max = 1, /* used for comparison only. */
 	tsd_state_purgatory = 2,
-	tsd_state_reincarnated = 3
+	tsd_state_reincarnated = 3,
+	tsd_state_uninitialized = 4
 };
+
 /* Manually limit tsd_state_t to a single byte. */
 typedef uint8_t tsd_state_t;