Fix tsd cleanup regressions.

Fix tsd cleanup regressions that were introduced in
5460aa6f66 (Convert all tsd variables to
reside in a single tsd structure.).  These regressions were twofold:

1) tsd_tryget() should never (and need never) return NULL.  Rename it to
   tsd_fetch() and simplify all callers.
2) tsd_*_set() must only be called when tsd is in the nominal state,
   because cleanup happens during the nominal-->purgatory transition,
   and re-initialization must not happen while in the purgatory state.
   Add tsd_nominal() and use it as needed.  Note that tsd_*{p,}_get()
   can still be used as long as no re-initialization that would require
   cleanup occurs.  This means that e.g. the thread_allocated counter
   can be updated unconditionally.

include/jemalloc/internal/private_symbols.txt

@@ -390,12 +390,14 @@ tsd_arena_set
 tsd_boot
 tsd_cleanup
 tsd_cleanup_wrapper
+tsd_fetch
 tsd_get
 tsd_get_wrapper
 tsd_initialized
 tsd_init_check_recursion
 tsd_init_finish
 tsd_init_head
+tsd_nominal
 tsd_quarantine_get
 tsd_quarantine_set
 tsd_set
@@ -411,7 +413,6 @@ tsd_thread_allocated_get
 tsd_thread_allocated_set
 tsd_thread_deallocated_get
 tsd_thread_deallocated_set
-tsd_tryget
 u2rz
 valgrind_freelike_block
 valgrind_make_mem_defined

include/jemalloc/internal/prof.h

@@ -331,8 +331,10 @@ prof_tdata_get(tsd_t *tsd, bool create)
 	tdata = tsd_prof_tdata_get(tsd);
 	if (create) {
 		if (unlikely(tdata == NULL)) {
-			tdata = prof_tdata_init(tsd);
-			tsd_prof_tdata_set(tsd, tdata);
+			if (tsd_nominal(tsd)) {
+				tdata = prof_tdata_init(tsd);
+				tsd_prof_tdata_set(tsd, tdata);
+			}
 		} else if (unlikely(tdata->expired)) {
 			tdata = prof_tdata_reinit(tsd, tdata);
 			tsd_prof_tdata_set(tsd, tdata);

include/jemalloc/internal/quarantine.h

@@ -49,8 +49,8 @@ quarantine_alloc_hook(void)
 
 	assert(config_fill && opt_quarantine);
 
-	tsd = tsd_tryget();
-	if (tsd != NULL && tsd_quarantine_get(tsd) == NULL)
+	tsd = tsd_fetch();
+	if (tsd_quarantine_get(tsd) == NULL && tsd_nominal(tsd))
 		tsd_quarantine_set(tsd, quarantine_init(tsd, LG_MAXOBJS_INIT));
 }
 #endif

include/jemalloc/internal/tcache.h

@@ -142,9 +142,8 @@ tcache_flush(void)
 
 	cassert(config_tcache);
 
-	tsd = tsd_tryget();
-	if (tsd != NULL)
-		tcache_cleanup(tsd);
+	tsd = tsd_fetch();
+	tcache_cleanup(tsd);
 }
 
 JEMALLOC_INLINE bool
@@ -155,9 +154,7 @@ tcache_enabled_get(void)
 
 	cassert(config_tcache);
 
-	tsd = tsd_tryget();
-	if (tsd == NULL)
-		return (false);
+	tsd = tsd_fetch();
 	tcache_enabled = tsd_tcache_enabled_get(tsd);
 	if (tcache_enabled == tcache_enabled_default) {
 		tcache_enabled = (tcache_enabled_t)opt_tcache;
@@ -175,9 +172,7 @@ tcache_enabled_set(bool enabled)
 
 	cassert(config_tcache);
 
-	tsd = tsd_tryget();
-	if (tsd == NULL)
-		return;
+	tsd = tsd_fetch();
 
 	tcache_enabled = (tcache_enabled_t)enabled;
 	tsd_tcache_enabled_set(tsd, tcache_enabled);
@@ -195,17 +190,11 @@ tcache_get(tsd_t *tsd, bool create)
 		return (NULL);
 	if (config_lazy_lock && !isthreaded)
 		return (NULL);
-	/*
-	 * If create is true, the caller has already assured that tsd is
-	 * non-NULL.
-	 */
-	if (!create && unlikely(tsd == NULL))
-		return (NULL);
 
 	tcache = tsd_tcache_get(tsd);
 	if (!create)
 		return (tcache);
-	if (unlikely(tcache == NULL)) {
+	if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
 		tcache = tcache_get_hard(tsd);
 		tsd_tcache_set(tsd, tcache);
 	}

include/jemalloc/internal/tsd.h

@@ -49,16 +49,19 @@ typedef enum {
  * Note that all of the functions deal in terms of (a_type *) rather than
  * (a_type)  so that it is possible to support non-pointer types (unlike
  * pthreads TSD).  example_tsd_cleanup() is passed an (a_type *) pointer that is
- * cast to (void *).  This means that the cleanup function needs to cast *and*
- * dereference the function argument, e.g.:
+ * cast to (void *).  This means that the cleanup function needs to cast the
+ * function argument to (a_type *), then dereference the resulting pointer to
+ * access fields, e.g.
  *
- *   bool
+ *   void
  *   example_tsd_cleanup(void *arg)
  *   {
- *           example_t *example = *(example_t **)arg;
+ *           example_t *example = (example_t *)arg;
  *
+ *           example->x = 42;
  *           [...]
- *           return ([want the cleanup function to be called again]);
+ *           if ([want the cleanup function to be called again])
+ *                   example_tsd_set(example);
  *   }
  *
  * If example_tsd_set() is called within example_tsd_cleanup(), it will be
@@ -468,7 +471,8 @@ void	tsd_cleanup(void *arg);
 #ifndef JEMALLOC_ENABLE_INLINE
 malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
 
-tsd_t	*tsd_tryget(void);
+tsd_t	*tsd_fetch(void);
+bool	tsd_nominal(tsd_t *tsd);
 #define	O(n, t)								\
 t	*tsd_##n##p_get(tsd_t *tsd);					\
 t	tsd_##n##_get(tsd_t *tsd);					\
@@ -481,50 +485,53 @@ MALLOC_TSD
 malloc_tsd_externs(, tsd_t)
 malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
 
-JEMALLOC_INLINE tsd_t *
-tsd_tryget(void)
+JEMALLOC_ALWAYS_INLINE tsd_t *
+tsd_fetch(void)
 {
-	tsd_t *tsd;
+	tsd_t *tsd = tsd_get();
 
-	tsd = tsd_get();
-	if (unlikely(tsd == NULL))
-		return (NULL);
-
-	if (likely(tsd->state == tsd_state_nominal))
-		return (tsd);
-	else if (tsd->state == tsd_state_uninitialized) {
-		tsd->state = tsd_state_nominal;
-		tsd_set(tsd);
-		return (tsd);
-	} else if (tsd->state == tsd_state_purgatory) {
-		tsd->state = tsd_state_reincarnated;
-		tsd_set(tsd);
-		return (NULL);
-	} else {
-		assert(tsd->state == tsd_state_reincarnated);
-		return (NULL);
+	if (unlikely(tsd->state != tsd_state_nominal)) {
+		if (tsd->state == tsd_state_uninitialized) {
+			tsd->state = tsd_state_nominal;
+			/* Trigger cleanup handler registration. */
+			tsd_set(tsd);
+		} else if (tsd->state == tsd_state_purgatory) {
+			tsd->state = tsd_state_reincarnated;
+			tsd_set(tsd);
+		} else
+			assert(tsd->state == tsd_state_reincarnated);
 	}
+
+	return (tsd);
+}
+
+JEMALLOC_INLINE bool
+tsd_nominal(tsd_t *tsd)
+{
+
+	return (tsd->state == tsd_state_nominal);
 }
 
 #define	O(n, t)								\
-JEMALLOC_INLINE t *							\
+JEMALLOC_ALWAYS_INLINE t *						\
 tsd_##n##p_get(tsd_t *tsd)						\
 {									\
 									\
 	return (&tsd->n);						\
 }									\
 									\
-JEMALLOC_INLINE t							\
+JEMALLOC_ALWAYS_INLINE t						\
 tsd_##n##_get(tsd_t *tsd)						\
 {									\
 									\
 	return (*tsd_##n##p_get(tsd));					\
 }									\
 									\
-JEMALLOC_INLINE void							\
+JEMALLOC_ALWAYS_INLINE void						\
 tsd_##n##_set(tsd_t *tsd, t n)						\
 {									\
 									\
+	assert(tsd->state == tsd_state_nominal);			\
 	tsd->n = n;							\
 }
 MALLOC_TSD