Convert all tsd variables to reside in a single tsd structure.

include/jemalloc/internal/arena.h

@@ -419,9 +419,9 @@ extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
 #endif
 bool	arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra, bool zero);
-void	*arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
+void	*arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
-    size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
+    size_t size, size_t extra, size_t alignment, bool zero,
-    bool try_tcache_dalloc);
+    bool try_tcache_alloc, bool try_tcache_dalloc);
 dss_prec_t	arena_dss_prec_get(arena_t *arena);
 bool	arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
 void	arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive,
@@ -485,10 +485,12 @@ unsigned	arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
     const void *ptr);
 prof_tctx_t	*arena_prof_tctx_get(const void *ptr);
 void	arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
-void	*arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache);
+void	*arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
+    bool try_tcache);
 size_t	arena_salloc(const void *ptr, bool demote);
-void	arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache);
+void	arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr,
-void	arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size,
+    bool try_tcache);
+void	arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
     bool try_tcache);
 #endif
 
@@ -1053,7 +1055,8 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
+arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
+    bool try_tcache)
 {
 	tcache_t *tcache;
 
@@ -1061,12 +1064,12 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
 	assert(size <= arena_maxclass);
 
 	if (likely(size <= SMALL_MAXCLASS)) {
-		if (likely(try_tcache) && likely((tcache = tcache_get(true)) !=
+		if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
-		    NULL))
+		    true)) != NULL))
 			return (tcache_alloc_small(tcache, size, zero));
 		else {
-			return (arena_malloc_small(choose_arena(arena), size,
+			return (arena_malloc_small(choose_arena(tsd, arena),
-			    zero));
+			    size, zero));
 		}
 	} else {
 		/*
@@ -1074,11 +1077,11 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
 		 * infinite recursion during tcache initialization.
 		 */
 		if (try_tcache && size <= tcache_maxclass && likely((tcache =
-		    tcache_get(true)) != NULL))
+		    tcache_get(tsd, true)) != NULL))
 			return (tcache_alloc_large(tcache, size, zero));
 		else {
-			return (arena_malloc_large(choose_arena(arena), size,
+			return (arena_malloc_large(choose_arena(tsd, arena),
-			    zero));
+			    size, zero));
 		}
 	}
 }
@@ -1128,7 +1131,7 @@ arena_salloc(const void *ptr, bool demote)
 }
 
 JEMALLOC_ALWAYS_INLINE void
-arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
+arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache)
 {
 	size_t pageind, mapbits;
 	tcache_t *tcache;
@@ -1141,8 +1144,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
 	assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
 	if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) {
 		/* Small allocation. */
-		if (likely(try_tcache) && likely((tcache = tcache_get(false)) !=
+		if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
-		    NULL)) {
+		    false)) != NULL)) {
 			size_t binind = arena_ptr_small_binind_get(ptr,
 			    mapbits);
 			tcache_dalloc_small(tcache, ptr, binind);
@@ -1154,7 +1157,7 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
 		assert(((uintptr_t)ptr & PAGE_MASK) == 0);
 
 		if (try_tcache && size <= tcache_maxclass && likely((tcache =
-		    tcache_get(false)) != NULL)) {
+		    tcache_get(tsd, false)) != NULL)) {
 			tcache_dalloc_large(tcache, ptr, size);
 		} else
 			arena_dalloc_large(chunk->arena, chunk, ptr);
@@ -1162,7 +1165,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
 }
 
 JEMALLOC_ALWAYS_INLINE void
-arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
+arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
+    bool try_tcache)
 {
 	tcache_t *tcache;
 
@@ -1171,8 +1175,8 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
 
 	if (likely(size <= SMALL_MAXCLASS)) {
 		/* Small allocation. */
-		if (likely(try_tcache) && likely((tcache = tcache_get(false)) !=
+		if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
-		    NULL)) {
+		    false)) != NULL)) {
 			size_t binind = small_size2bin(size);
 			tcache_dalloc_small(tcache, ptr, binind);
 		} else {
@@ -1184,7 +1188,7 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
 		assert(((uintptr_t)ptr & PAGE_MASK) == 0);
 
 		if (try_tcache && size <= tcache_maxclass && (tcache =
-		    tcache_get(false)) != NULL) {
+		    tcache_get(tsd, false)) != NULL) {
 			tcache_dalloc_large(tcache, ptr, size);
 		} else
 			arena_dalloc_large(chunk->arena, chunk, ptr);

include/jemalloc/internal/ckh.h

@@ -66,13 +66,13 @@ struct ckh_s {
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-bool	ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
+bool	ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
     ckh_keycomp_t *keycomp);
-void	ckh_delete(ckh_t *ckh);
+void	ckh_delete(tsd_t *tsd, ckh_t *ckh);
 size_t	ckh_count(ckh_t *ckh);
 bool	ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
-bool	ckh_insert(ckh_t *ckh, const void *key, const void *data);
+bool	ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
-bool	ckh_remove(ckh_t *ckh, const void *searchkey, void **key,
+bool	ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
     void **data);
 bool	ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data);
 void	ckh_string_hash(const void *key, size_t r_hash[2]);

include/jemalloc/internal/huge.h

@@ -9,12 +9,14 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-void	*huge_malloc(arena_t *arena, size_t size, bool zero);
+void	*huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero);
-void	*huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
+void	*huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
+    bool zero);
 bool	huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra);
-void	*huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
+void	*huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
-    size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc);
+    size_t size, size_t extra, size_t alignment, bool zero,
+    bool try_tcache_dalloc);
 #ifdef JEMALLOC_JET
 typedef void (huge_dalloc_junk_t)(void *, size_t);
 extern huge_dalloc_junk_t *huge_dalloc_junk;

include/jemalloc/internal/jemalloc_internal.h.in

@@ -350,7 +350,6 @@ static const bool config_ivsalloc =
 #include "jemalloc/internal/stats.h"
 #include "jemalloc/internal/ctl.h"
 #include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/tsd.h"
 #include "jemalloc/internal/mb.h"
 #include "jemalloc/internal/bitmap.h"
 #include "jemalloc/internal/extent.h"
@@ -364,15 +363,7 @@ static const bool config_ivsalloc =
 #include "jemalloc/internal/quarantine.h"
 #include "jemalloc/internal/prof.h"
 
-typedef struct {
+#include "jemalloc/internal/tsd.h"
-	uint64_t	allocated;
-	uint64_t	deallocated;
-} thread_allocated_t;
-/*
- * The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro
- * argument.
- */
-#define	THREAD_ALLOCATED_INITIALIZER	JEMALLOC_ARG_CONCAT({0, 0})
 
 #undef JEMALLOC_H_STRUCTS
 /******************************************************************************/
@@ -407,8 +398,10 @@ extern unsigned		narenas_total;
 extern unsigned		narenas_auto; /* Read-only after initialization. */
 
 arena_t	*arenas_extend(unsigned ind);
-void	arenas_cleanup(void *arg);
+arena_t	*choose_arena_hard(tsd_t *tsd);
-arena_t	*choose_arena_hard(void);
+void	thread_allocated_cleanup(tsd_t *tsd);
+void	thread_deallocated_cleanup(tsd_t *tsd);
+void	arena_cleanup(tsd_t *tsd);
 void	jemalloc_prefork(void);
 void	jemalloc_postfork_parent(void);
 void	jemalloc_postfork_child(void);
@@ -422,7 +415,6 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/stats.h"
 #include "jemalloc/internal/ctl.h"
 #include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/tsd.h"
 #include "jemalloc/internal/mb.h"
 #include "jemalloc/internal/bitmap.h"
 #include "jemalloc/internal/extent.h"
@@ -435,6 +427,7 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/hash.h"
 #include "jemalloc/internal/quarantine.h"
 #include "jemalloc/internal/prof.h"
+#include "jemalloc/internal/tsd.h"
 
 #undef JEMALLOC_H_EXTERNS
 /******************************************************************************/
@@ -465,23 +458,13 @@ void	jemalloc_postfork_child(void);
 #undef JEMALLOC_ARENA_INLINE_A
 
 #ifndef JEMALLOC_ENABLE_INLINE
-malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
-
 size_t	s2u(size_t size);
 size_t	sa2u(size_t size, size_t alignment);
 unsigned	narenas_total_get(void);
-arena_t	*choose_arena(arena_t *arena);
+arena_t	*choose_arena(tsd_t *tsd, arena_t *arena);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
-/*
- * Map of pthread_self() --> arenas[???], used for selecting an arena to use
- * for allocations.
- */
-malloc_tsd_externs(arenas, arena_t *)
-malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL,
-    arenas_cleanup)
-
 /*
  * Compute usable size that would result from allocating an object with the
  * specified size.
@@ -589,15 +572,15 @@ narenas_total_get(void)
 
 /* Choose an arena based on a per-thread value. */
 JEMALLOC_INLINE arena_t *
-choose_arena(arena_t *arena)
+choose_arena(tsd_t *tsd, arena_t *arena)
 {
 	arena_t *ret;
 
 	if (arena != NULL)
 		return (arena);
 
-	if ((ret = *arenas_tsd_get()) == NULL) {
+	if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) {
-		ret = choose_arena_hard();
+		ret = choose_arena_hard(tsd);
 		assert(ret != NULL);
 	}
 
@@ -622,72 +605,72 @@ choose_arena(arena_t *arena)
 #include "jemalloc/internal/quarantine.h"
 
 #ifndef JEMALLOC_ENABLE_INLINE
-void	*imalloct(size_t size, bool try_tcache, arena_t *arena);
+void	*imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
-void	*imalloc(size_t size);
+void	*imalloc(tsd_t *tsd, size_t size);
-void	*icalloct(size_t size, bool try_tcache, arena_t *arena);
+void	*icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
-void	*icalloc(size_t size);
+void	*icalloc(tsd_t *tsd, size_t size);
-void	*ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
+void	*ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
-    arena_t *arena);
+    bool try_tcache, arena_t *arena);
-void	*ipalloc(size_t usize, size_t alignment, bool zero);
+void	*ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
 size_t	isalloc(const void *ptr, bool demote);
 size_t	ivsalloc(const void *ptr, bool demote);
 size_t	u2rz(size_t usize);
 size_t	p2rz(const void *ptr);
-void	idalloct(void *ptr, bool try_tcache);
+void	idalloct(tsd_t *tsd, void *ptr, bool try_tcache);
-void	isdalloct(void *ptr, size_t size, bool try_tcache);
+void	isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
-void	idalloc(void *ptr);
+void	idalloc(tsd_t *tsd, void *ptr);
-void	iqalloc(void *ptr, bool try_tcache);
+void	iqalloc(tsd_t *tsd, void *ptr, bool try_tcache);
-void	isqalloc(void *ptr, size_t size, bool try_tcache);
+void	isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
-void	*iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
+void	*iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
+    size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
-    arena_t *arena);
+    bool try_tcache_dalloc, arena_t *arena);
-void	*iralloct(void *ptr, size_t size, size_t alignment, bool zero,
+void	*iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
-    bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
+    bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
-void	*iralloc(void *ptr, size_t size, size_t alignment, bool zero);
+void	*iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
+    bool zero);
 bool	ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
     bool zero);
-malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
 JEMALLOC_ALWAYS_INLINE void *
-imalloct(size_t size, bool try_tcache, arena_t *arena)
+imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
 {
 
 	assert(size != 0);
 
 	if (size <= arena_maxclass)
-		return (arena_malloc(arena, size, false, try_tcache));
+		return (arena_malloc(tsd, arena, size, false, try_tcache));
 	else
-		return (huge_malloc(arena, size, false));
+		return (huge_malloc(tsd, arena, size, false));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-imalloc(size_t size)
+imalloc(tsd_t *tsd, size_t size)
 {
 
-	return (imalloct(size, true, NULL));
+	return (imalloct(tsd, size, true, NULL));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-icalloct(size_t size, bool try_tcache, arena_t *arena)
+icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
 {
 
 	if (size <= arena_maxclass)
-		return (arena_malloc(arena, size, true, try_tcache));
+		return (arena_malloc(tsd, arena, size, true, try_tcache));
 	else
-		return (huge_malloc(arena, size, true));
+		return (huge_malloc(tsd, arena, size, true));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-icalloc(size_t size)
+icalloc(tsd_t *tsd, size_t size)
 {
 
-	return (icalloct(size, true, NULL));
+	return (icalloct(tsd, size, true, NULL));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
+ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache,
     arena_t *arena)
 {
 	void *ret;
@@ -696,15 +679,15 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
 	assert(usize == sa2u(usize, alignment));
 
 	if (usize <= arena_maxclass && alignment <= PAGE)
-		ret = arena_malloc(arena, usize, zero, try_tcache);
+		ret = arena_malloc(tsd, arena, usize, zero, try_tcache);
 	else {
 		if (usize <= arena_maxclass) {
-			ret = arena_palloc(choose_arena(arena), usize,
+			ret = arena_palloc(choose_arena(tsd, arena), usize,
 			    alignment, zero);
 		} else if (alignment <= chunksize)
-			ret = huge_malloc(arena, usize, zero);
+			ret = huge_malloc(tsd, arena, usize, zero);
 		else
-			ret = huge_palloc(arena, usize, alignment, zero);
+			ret = huge_palloc(tsd, arena, usize, alignment, zero);
 	}
 
 	assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
@@ -712,10 +695,10 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-ipalloc(size_t usize, size_t alignment, bool zero)
+ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero)
 {
 
-	return (ipalloct(usize, alignment, zero, true, NULL));
+	return (ipalloct(tsd, usize, alignment, zero, true, NULL));
 }
 
 /*
@@ -776,7 +759,7 @@ p2rz(const void *ptr)
 }
 
 JEMALLOC_ALWAYS_INLINE void
-idalloct(void *ptr, bool try_tcache)
+idalloct(tsd_t *tsd, void *ptr, bool try_tcache)
 {
 	arena_chunk_t *chunk;
 
@@ -784,13 +767,13 @@ idalloct(void *ptr, bool try_tcache)
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	if (chunk != ptr)
-		arena_dalloc(chunk, ptr, try_tcache);
+		arena_dalloc(tsd, chunk, ptr, try_tcache);
 	else
 		huge_dalloc(ptr);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-isdalloct(void *ptr, size_t size, bool try_tcache)
+isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
 {
 	arena_chunk_t *chunk;
 
@@ -798,42 +781,42 @@ isdalloct(void *ptr, size_t size, bool try_tcache)
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	if (chunk != ptr)
-		arena_sdalloc(chunk, ptr, size, try_tcache);
+		arena_sdalloc(tsd, chunk, ptr, size, try_tcache);
 	else
 		huge_dalloc(ptr);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-idalloc(void *ptr)
+idalloc(tsd_t *tsd, void *ptr)
 {
 
-	idalloct(ptr, true);
+	idalloct(tsd, ptr, true);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-iqalloc(void *ptr, bool try_tcache)
+iqalloc(tsd_t *tsd, void *ptr, bool try_tcache)
 {
 
 	if (config_fill && unlikely(opt_quarantine))
-		quarantine(ptr);
+		quarantine(tsd, ptr);
 	else
-		idalloct(ptr, try_tcache);
+		idalloct(tsd, ptr, try_tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-isqalloc(void *ptr, size_t size, bool try_tcache)
+isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
 {
 
 	if (config_fill && unlikely(opt_quarantine))
-		quarantine(ptr);
+		quarantine(tsd, ptr);
 	else
-		isdalloct(ptr, size, try_tcache);
+		isdalloct(tsd, ptr, size, try_tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
+iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
+    size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
-    arena_t *arena)
+    bool try_tcache_dalloc, arena_t *arena)
 {
 	void *p;
 	size_t usize, copysize;
@@ -841,7 +824,7 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
 	usize = sa2u(size + extra, alignment);
 	if (usize == 0)
 		return (NULL);
-	p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
+	p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, arena);
 	if (p == NULL) {
 		if (extra == 0)
 			return (NULL);
@@ -849,7 +832,8 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
 		usize = sa2u(size, alignment);
 		if (usize == 0)
 			return (NULL);
-		p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
+		p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc,
+		    arena);
 		if (p == NULL)
 			return (NULL);
 	}
@@ -859,12 +843,12 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
 	 */
 	copysize = (size < oldsize) ? size : oldsize;
 	memcpy(p, ptr, copysize);
-	iqalloc(ptr, try_tcache_dalloc);
+	iqalloc(tsd, ptr, try_tcache_dalloc);
 	return (p);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iralloct(void *ptr, size_t size, size_t alignment, bool zero,
+iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero,
     bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
 {
 	size_t oldsize;
@@ -880,24 +864,24 @@ iralloct(void *ptr, size_t size, size_t alignment, bool zero,
 		 * Existing object alignment is inadequate; allocate new space
 		 * and copy.
 		 */
-		return (iralloct_realign(ptr, oldsize, size, 0, alignment, zero,
+		return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment,
-		    try_tcache_alloc, try_tcache_dalloc, arena));
+		    zero, try_tcache_alloc, try_tcache_dalloc, arena));
 	}
 
 	if (size <= arena_maxclass) {
-		return (arena_ralloc(arena, ptr, oldsize, size, 0, alignment,
+		return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0,
-		    zero, try_tcache_alloc, try_tcache_dalloc));
+		    alignment, zero, try_tcache_alloc, try_tcache_dalloc));
 	} else {
-		return (huge_ralloc(arena, ptr, oldsize, size, 0, alignment,
+		return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0,
-		    zero, try_tcache_dalloc));
+		    alignment, zero, try_tcache_dalloc));
 	}
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iralloc(void *ptr, size_t size, size_t alignment, bool zero)
+iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero)
 {
 
-	return (iralloct(ptr, size, alignment, zero, true, true, NULL));
+	return (iralloct(tsd, ptr, size, alignment, zero, true, true, NULL));
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -920,10 +904,6 @@ ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
 	else
 		return (huge_ralloc_no_move(ptr, oldsize, size, extra));
 }
-
-malloc_tsd_externs(thread_allocated, thread_allocated_t)
-malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t,
-    THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
 #endif
 
 #include "jemalloc/internal/prof.h"

include/jemalloc/internal/private_symbols.txt

@@ -8,6 +8,7 @@ arena_bitselm_get
 arena_boot
 arena_chunk_alloc_huge
 arena_chunk_dalloc_huge
+arena_cleanup
 arena_dalloc
 arena_dalloc_bin
 arena_dalloc_bin_locked
@@ -65,19 +66,9 @@ arena_sdalloc
 arena_stats_merge
 arena_tcache_fill_small
 arenas
-arenas_booted
 arenas_cleanup
 arenas_extend
-arenas_initialized
 arenas_lock
-arenas_tls
-arenas_tsd
-arenas_tsd_boot
-arenas_tsd_cleanup_wrapper
-arenas_tsd_get
-arenas_tsd_get_wrapper
-arenas_tsd_init_head
-arenas_tsd_set
 atomic_add_u
 atomic_add_uint32
 atomic_add_uint64
@@ -317,37 +308,17 @@ prof_sample_accum_update
 prof_sample_threshold_update
 prof_tctx_get
 prof_tctx_set
-prof_tdata_booted
 prof_tdata_cleanup
 prof_tdata_get
 prof_tdata_init
-prof_tdata_initialized
-prof_tdata_tls
-prof_tdata_tsd
-prof_tdata_tsd_boot
-prof_tdata_tsd_cleanup_wrapper
-prof_tdata_tsd_get
-prof_tdata_tsd_get_wrapper
-prof_tdata_tsd_init_head
-prof_tdata_tsd_set
 prof_thread_active_get
 prof_thread_active_set
 prof_thread_name_get
 prof_thread_name_set
 quarantine
 quarantine_alloc_hook
-quarantine_boot
-quarantine_booted
 quarantine_cleanup
 quarantine_init
-quarantine_tls
-quarantine_tsd
-quarantine_tsd_boot
-quarantine_tsd_cleanup_wrapper
-quarantine_tsd_get
-quarantine_tsd_get_wrapper
-quarantine_tsd_init_head
-quarantine_tsd_set
 register_zone
 rtree_delete
 rtree_get
@@ -386,55 +357,52 @@ tcache_arena_dissociate
 tcache_bin_flush_large
 tcache_bin_flush_small
 tcache_bin_info
-tcache_boot0
+tcache_boot
-tcache_boot1
+tcache_cleanup
-tcache_booted
 tcache_create
 tcache_dalloc_large
 tcache_dalloc_small
-tcache_destroy
+tcache_enabled_cleanup
-tcache_enabled_booted
 tcache_enabled_get
-tcache_enabled_initialized
 tcache_enabled_set
-tcache_enabled_tls
-tcache_enabled_tsd
-tcache_enabled_tsd_boot
-tcache_enabled_tsd_cleanup_wrapper
-tcache_enabled_tsd_get
-tcache_enabled_tsd_get_wrapper
-tcache_enabled_tsd_init_head
-tcache_enabled_tsd_set
 tcache_event
 tcache_event_hard
 tcache_flush
 tcache_get
 tcache_get_hard
-tcache_initialized
 tcache_maxclass
 tcache_salloc
 tcache_stats_merge
-tcache_thread_cleanup
+thread_allocated_cleanup
-tcache_tls
+thread_deallocated_cleanup
-tcache_tsd
+tsd_booted
-tcache_tsd_boot
+tsd_arena_get
-tcache_tsd_cleanup_wrapper
+tsd_arena_set
-tcache_tsd_get
+tsd_boot
-tcache_tsd_get_wrapper
+tsd_cleanup
-tcache_tsd_init_head
+tsd_cleanup_wrapper
-tcache_tsd_set
+tsd_get
-thread_allocated_booted
+tsd_get_wrapper
-thread_allocated_initialized
+tsd_initialized
-thread_allocated_tls
-thread_allocated_tsd
-thread_allocated_tsd_boot
-thread_allocated_tsd_cleanup_wrapper
-thread_allocated_tsd_get
-thread_allocated_tsd_get_wrapper
-thread_allocated_tsd_init_head
-thread_allocated_tsd_set
 tsd_init_check_recursion
 tsd_init_finish
+tsd_init_head
+tsd_quarantine_get
+tsd_quarantine_set
+tsd_set
+tsd_tcache_enabled_get
+tsd_tcache_enabled_set
+tsd_tcache_get
+tsd_tcache_set
+tsd_tls
+tsd_tsd
+tsd_prof_tdata_get
+tsd_prof_tdata_set
+tsd_thread_allocated_get
+tsd_thread_allocated_set
+tsd_thread_deallocated_get
+tsd_thread_deallocated_set
+tsd_tryget
 u2rz
 valgrind_freelike_block
 valgrind_make_mem_defined

include/jemalloc/internal/prof.h

@@ -248,13 +248,13 @@ extern uint64_t	prof_interval;
  */
 extern size_t	lg_prof_sample;
 
-void	prof_alloc_rollback(prof_tctx_t *tctx, bool updated);
+void	prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
 void	prof_malloc_sample_object(const void *ptr, size_t usize,
     prof_tctx_t *tctx);
-void	prof_free_sampled_object(size_t usize, prof_tctx_t *tctx);
+void	prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
 void	bt_init(prof_bt_t *bt, void **vec);
 void	prof_backtrace(prof_bt_t *bt);
-prof_tctx_t	*prof_lookup(prof_bt_t *bt);
+prof_tctx_t	*prof_lookup(tsd_t *tsd, prof_bt_t *bt);
 #ifdef JEMALLOC_JET
 size_t	prof_bt_count(void);
 typedef int (prof_dump_open_t)(bool, const char *);
@@ -263,12 +263,12 @@ extern prof_dump_open_t *prof_dump_open;
 void	prof_idump(void);
 bool	prof_mdump(const char *filename);
 void	prof_gdump(void);
-prof_tdata_t	*prof_tdata_init(void);
+prof_tdata_t	*prof_tdata_init(tsd_t *tsd);
-prof_tdata_t	*prof_tdata_reinit(prof_tdata_t *tdata);
+prof_tdata_t	*prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
-void	prof_reset(size_t lg_sample);
+void	prof_reset(tsd_t *tsd, size_t lg_sample);
-void	prof_tdata_cleanup(void *arg);
+void	prof_tdata_cleanup(tsd_t *tsd);
 const char	*prof_thread_name_get(void);
-bool	prof_thread_name_set(const char *thread_name);
+bool	prof_thread_name_set(tsd_t *tsd, const char *thread_name);
 bool	prof_thread_active_get(void);
 bool	prof_thread_active_set(bool active);
 void	prof_boot0(void);
@@ -284,43 +284,38 @@ void	prof_sample_threshold_update(prof_tdata_t *tdata);
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
-malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
+prof_tdata_t	*prof_tdata_get(tsd_t *tsd, bool create);
-
+bool	prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
-prof_tdata_t	*prof_tdata_get(bool create);
-bool	prof_sample_accum_update(size_t usize, bool commit,
     prof_tdata_t **tdata_out);
-prof_tctx_t	*prof_alloc_prep(size_t usize, bool update);
+prof_tctx_t	*prof_alloc_prep(tsd_t *tsd, size_t usize, bool update);
 prof_tctx_t	*prof_tctx_get(const void *ptr);
 void	prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
 void	prof_malloc_sample_object(const void *ptr, size_t usize,
     prof_tctx_t *tctx);
 void	prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx);
-void	prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx,
+void	prof_realloc(tsd_t *tsd, const void *ptr, size_t usize,
-    bool updated, size_t old_usize, prof_tctx_t *old_tctx);
+    prof_tctx_t *tctx, bool updated, size_t old_usize, prof_tctx_t *old_tctx);
-void	prof_free(const void *ptr, size_t usize);
+void	prof_free(tsd_t *tsd, const void *ptr, size_t usize);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
-/* Thread-specific backtrace cache, used to reduce bt2gctx contention. */
-malloc_tsd_externs(prof_tdata, prof_tdata_t *)
-malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
-    prof_tdata_cleanup)
-
 JEMALLOC_INLINE prof_tdata_t *
-prof_tdata_get(bool create)
+prof_tdata_get(tsd_t *tsd, bool create)
 {
 	prof_tdata_t *tdata;
 
 	cassert(config_prof);
 
-	tdata = *prof_tdata_tsd_get();
+	tdata = tsd_prof_tdata_get(tsd);
 	if (create) {
-		if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {
+		if (unlikely(tdata == NULL)) {
-			if (tdata == NULL)
+			tdata = prof_tdata_init(tsd);
-				tdata = prof_tdata_init();
+			tsd_prof_tdata_set(tsd, tdata);
-		} else if (tdata->state == prof_tdata_state_expired)
+		} else if (unlikely(tdata->state == prof_tdata_state_expired)) {
-			tdata = prof_tdata_reinit(tdata);
+			tdata = prof_tdata_reinit(tsd, tdata);
-		assert((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX ||
+			tsd_prof_tdata_set(tsd, tdata);
+		}
+		assert(tdata == NULL ||
 		    tdata->state == prof_tdata_state_attached);
 	}
 
@@ -363,13 +358,14 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
 }
 
 JEMALLOC_INLINE bool
-prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out)
+prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
+    prof_tdata_t **tdata_out)
 {
 	prof_tdata_t *tdata;
 
 	cassert(config_prof);
 
-	tdata = prof_tdata_get(true);
+	tdata = prof_tdata_get(tsd, true);
 	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		tdata = NULL;
 
@@ -392,7 +388,7 @@ prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out)
 }
 
 JEMALLOC_INLINE prof_tctx_t *
-prof_alloc_prep(size_t usize, bool update)
+prof_alloc_prep(tsd_t *tsd, size_t usize, bool update)
 {
 	prof_tctx_t *ret;
 	prof_tdata_t *tdata;
@@ -400,13 +396,13 @@ prof_alloc_prep(size_t usize, bool update)
 
 	assert(usize == s2u(usize));
 
-	if (!opt_prof_active || likely(prof_sample_accum_update(usize, update,
+	if (!opt_prof_active || likely(prof_sample_accum_update(tsd, usize,
-	    &tdata)))
+	    update, &tdata)))
 		ret = (prof_tctx_t *)(uintptr_t)1U;
 	else {
 		bt_init(&bt, tdata->vec);
 		prof_backtrace(&bt);
-		ret = prof_lookup(&bt);
+		ret = prof_lookup(tsd, &bt);
 	}
 
 	return (ret);
@@ -427,8 +423,8 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx)
 }
 
 JEMALLOC_INLINE void
-prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
+prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
-    size_t old_usize, prof_tctx_t *old_tctx)
+    bool updated, size_t old_usize, prof_tctx_t *old_tctx)
 {
 
 	cassert(config_prof);
@@ -436,7 +432,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
 
 	if (!updated && ptr != NULL) {
 		assert(usize == isalloc(ptr, true));
-		if (prof_sample_accum_update(usize, true, NULL)) {
+		if (prof_sample_accum_update(tsd, usize, true, NULL)) {
 			/*
 			 * Don't sample.  The usize passed to PROF_ALLOC_PREP()
 			 * was larger than what actually got allocated, so a
@@ -449,7 +445,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
 	}
 
 	if (unlikely((uintptr_t)old_tctx > (uintptr_t)1U))
-		prof_free_sampled_object(old_usize, old_tctx);
+		prof_free_sampled_object(tsd, old_usize, old_tctx);
 	if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
 		prof_malloc_sample_object(ptr, usize, tctx);
 	else
@@ -457,7 +453,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
 }
 
 JEMALLOC_INLINE void
-prof_free(const void *ptr, size_t usize)
+prof_free(tsd_t *tsd, const void *ptr, size_t usize)
 {
 	prof_tctx_t *tctx = prof_tctx_get(ptr);
 
@@ -465,7 +461,7 @@ prof_free(const void *ptr, size_t usize)
 	assert(usize == isalloc(ptr, true));
 
 	if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
-		prof_free_sampled_object(usize, tctx);
+		prof_free_sampled_object(tsd, usize, tctx);
 }
 #endif
 

include/jemalloc/internal/quarantine.h

@@ -29,36 +29,29 @@ struct quarantine_s {
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-quarantine_t	*quarantine_init(size_t lg_maxobjs);
+quarantine_t	*quarantine_init(tsd_t *tsd, size_t lg_maxobjs);
-void	quarantine(void *ptr);
+void	quarantine(tsd_t *tsd, void *ptr);
-void	quarantine_cleanup(void *arg);
+void	quarantine_cleanup(tsd_t *tsd);
-bool	quarantine_boot(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
-malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *)
-
 void	quarantine_alloc_hook(void);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_))
-malloc_tsd_externs(quarantine, quarantine_t *)
-malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL,
-    quarantine_cleanup)
-
 JEMALLOC_ALWAYS_INLINE void
 quarantine_alloc_hook(void)
 {
-	quarantine_t *quarantine;
+	tsd_t *tsd;
 
 	assert(config_fill && opt_quarantine);
 
-	quarantine = *quarantine_tsd_get();
+	tsd = tsd_tryget();
-	if (quarantine == NULL)
+	if (tsd != NULL && tsd_quarantine_get(tsd) == NULL)
-		quarantine_init(LG_MAXOBJS_INIT);
+		tsd_quarantine_set(tsd, quarantine_init(tsd, LG_MAXOBJS_INIT));
 }
 #endif
 

include/jemalloc/internal/tcache.h

@@ -110,26 +110,22 @@ void	tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
     tcache_t *tcache);
 void	tcache_arena_associate(tcache_t *tcache, arena_t *arena);
 void	tcache_arena_dissociate(tcache_t *tcache);
-tcache_t *tcache_get_hard(tcache_t *tcache, bool create);
+tcache_t *tcache_get_hard(tsd_t *tsd);
 tcache_t *tcache_create(arena_t *arena);
-void	tcache_destroy(tcache_t *tcache);
+void	tcache_cleanup(tsd_t *tsd);
-void	tcache_thread_cleanup(void *arg);
+void	tcache_enabled_cleanup(tsd_t *tsd);
 void	tcache_stats_merge(tcache_t *tcache, arena_t *arena);
-bool	tcache_boot0(void);
+bool	tcache_boot(void);
-bool	tcache_boot1(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
-malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *)
-malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t)
-
 void	tcache_event(tcache_t *tcache);
 void	tcache_flush(void);
 bool	tcache_enabled_get(void);
-tcache_t *tcache_get(bool create);
+tcache_t *tcache_get(tsd_t *tsd, bool create);
 void	tcache_enabled_set(bool enabled);
 void	*tcache_alloc_easy(tcache_bin_t *tbin);
 void	*tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
@@ -139,41 +135,33 @@ void	tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
-/* Map of thread-specific caches. */
-malloc_tsd_externs(tcache, tcache_t *)
-malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL,
-    tcache_thread_cleanup)
-/* Per thread flag that allows thread caches to be disabled. */
-malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
-malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t,
-    tcache_enabled_default, malloc_tsd_no_cleanup)
-
 JEMALLOC_INLINE void
 tcache_flush(void)
 {
-	tcache_t *tcache;
+	tsd_t *tsd;
 
 	cassert(config_tcache);
 
-	tcache = *tcache_tsd_get();
+	tsd = tsd_tryget();
-	if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)
+	if (tsd != NULL)
-		return;
+		tcache_cleanup(tsd);
-	tcache_destroy(tcache);
-	tcache = NULL;
-	tcache_tsd_set(&tcache);
 }
 
 JEMALLOC_INLINE bool
 tcache_enabled_get(void)
 {
+	tsd_t *tsd;
 	tcache_enabled_t tcache_enabled;
 
 	cassert(config_tcache);
 
-	tcache_enabled = *tcache_enabled_tsd_get();
+	tsd = tsd_tryget();
+	if (tsd == NULL)
+		return (false);
+	tcache_enabled = tsd_tcache_enabled_get(tsd);
 	if (tcache_enabled == tcache_enabled_default) {
 		tcache_enabled = (tcache_enabled_t)opt_tcache;
-		tcache_enabled_tsd_set(&tcache_enabled);
+		tsd_tcache_enabled_set(tsd, tcache_enabled);
 	}
 
 	return ((bool)tcache_enabled);
@@ -182,33 +170,24 @@ tcache_enabled_get(void)
 JEMALLOC_INLINE void
 tcache_enabled_set(bool enabled)
 {
+	tsd_t *tsd;
 	tcache_enabled_t tcache_enabled;
-	tcache_t *tcache;
 
 	cassert(config_tcache);
 
+	tsd = tsd_tryget();
+	if (tsd == NULL)
+		return;
+
 	tcache_enabled = (tcache_enabled_t)enabled;
-	tcache_enabled_tsd_set(&tcache_enabled);
+	tsd_tcache_enabled_set(tsd, tcache_enabled);
-	tcache = *tcache_tsd_get();
+
-	if (enabled) {
+	if (!enabled)
-		if (tcache == TCACHE_STATE_DISABLED) {
+		tcache_cleanup(tsd);
-			tcache = NULL;
-			tcache_tsd_set(&tcache);
-		}
-	} else /* disabled */ {
-		if (tcache > TCACHE_STATE_MAX) {
-			tcache_destroy(tcache);
-			tcache = NULL;
-		}
-		if (tcache == NULL) {
-			tcache = TCACHE_STATE_DISABLED;
-			tcache_tsd_set(&tcache);
-		}
-	}
 }
 
 JEMALLOC_ALWAYS_INLINE tcache_t *
-tcache_get(bool create)
+tcache_get(tsd_t *tsd, bool create)
 {
 	tcache_t *tcache;
 
@@ -216,12 +195,19 @@ tcache_get(bool create)
 		return (NULL);
 	if (config_lazy_lock && isthreaded == false)
 		return (NULL);
-
+	/*
-	tcache = *tcache_tsd_get();
+	 * If create is true, the caller has already assured that tsd is
-	if (unlikely((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)) {
+	 * non-NULL.
-		if (tcache == TCACHE_STATE_DISABLED)
+	 */
+	if (!create && unlikely(tsd == NULL))
 		return (NULL);
-		tcache = tcache_get_hard(tcache, create);
+
+	tcache = tsd_tcache_get(tsd);
+	if (!create)
+		return (tcache);
+	if (unlikely(tcache == NULL)) {
+		tcache = tcache_get_hard(tsd);
+		tsd_tcache_set(tsd, tcache);
 	}
 
 	return (tcache);

include/jemalloc/internal/tsd.h

@@ -12,6 +12,15 @@ typedef struct tsd_init_block_s tsd_init_block_t;
 typedef struct tsd_init_head_s tsd_init_head_t;
 #endif
 
+typedef struct tsd_s tsd_t;
+
+typedef enum {
+	tsd_state_uninitialized,
+	tsd_state_nominal,
+	tsd_state_purgatory,
+	tsd_state_reincarnated
+} tsd_state_t;
+
 /*
  * TLS/TSD-agnostic macro-based implementation of thread-specific data.  There
  * are four macros that support (at least) three use cases: file-private,
@@ -24,11 +33,11 @@ typedef struct tsd_init_head_s tsd_init_head_t;
  *           int y;
  *   } example_t;
  *   #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
- *   malloc_tsd_protos(, example, example_t *)
+ *   malloc_tsd_protos(, example_, example_t *)
- *   malloc_tsd_externs(example, example_t *)
+ *   malloc_tsd_externs(example_, example_t *)
  * In example.c:
- *   malloc_tsd_data(, example, example_t *, EX_INITIALIZER)
+ *   malloc_tsd_data(, example_, example_t *, EX_INITIALIZER)
- *   malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER,
+ *   malloc_tsd_funcs(, example_, example_t *, EX_INITIALIZER,
  *       example_tsd_cleanup)
  *
  * The result is a set of generated functions, e.g.:
@@ -43,15 +52,13 @@ typedef struct tsd_init_head_s tsd_init_head_t;
  * cast to (void *).  This means that the cleanup function needs to cast *and*
  * dereference the function argument, e.g.:
  *
- *   void
+ *   bool
  *   example_tsd_cleanup(void *arg)
  *   {
  *           example_t *example = *(example_t **)arg;
  *
  *           [...]
- *           if ([want the cleanup function to be called again]) {
+ *           return ([want the cleanup function to be called again]);
- *                   example_tsd_set(&example);
- *           }
  *   }
  *
  * If example_tsd_set() is called within example_tsd_cleanup(), it will be
@@ -63,60 +70,60 @@ typedef struct tsd_init_head_s tsd_init_head_t;
 /* malloc_tsd_protos(). */
 #define	malloc_tsd_protos(a_attr, a_name, a_type)			\
 a_attr bool								\
-a_name##_tsd_boot(void);						\
+a_name##tsd_boot(void);							\
 a_attr a_type *								\
-a_name##_tsd_get(void);							\
+a_name##tsd_get(void);							\
 a_attr void								\
-a_name##_tsd_set(a_type *val);
+a_name##tsd_set(a_type *val);
 
 /* malloc_tsd_externs(). */
 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
 #define	malloc_tsd_externs(a_name, a_type)				\
-extern __thread a_type	a_name##_tls;					\
+extern __thread a_type	a_name##tsd_tls;				\
-extern __thread bool	a_name##_initialized;				\
+extern __thread bool	a_name##tsd_initialized;			\
-extern bool		a_name##_booted;
+extern bool		a_name##tsd_booted;
 #elif (defined(JEMALLOC_TLS))
 #define	malloc_tsd_externs(a_name, a_type)				\
-extern __thread a_type	a_name##_tls;					\
+extern __thread a_type	a_name##tsd_tls;				\
-extern pthread_key_t	a_name##_tsd;					\
+extern pthread_key_t	a_name##tsd_tsd;				\
-extern bool		a_name##_booted;
+extern bool		a_name##tsd_booted;
 #elif (defined(_WIN32))
 #define	malloc_tsd_externs(a_name, a_type)				\
-extern DWORD		a_name##_tsd;					\
+extern DWORD		a_name##tsd_tsd;				\
-extern bool		a_name##_booted;
+extern bool		a_name##tsd_booted;
 #else
 #define	malloc_tsd_externs(a_name, a_type)				\
-extern pthread_key_t	a_name##_tsd;					\
+extern pthread_key_t	a_name##tsd_tsd;				\
-extern tsd_init_head_t	a_name##_tsd_init_head;				\
+extern tsd_init_head_t	a_name##tsd_init_head;				\
-extern bool		a_name##_booted;
+extern bool		a_name##tsd_booted;
 #endif
 
 /* malloc_tsd_data(). */
 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
 #define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
 a_attr __thread a_type JEMALLOC_TLS_MODEL				\
-    a_name##_tls = a_initializer;					\
+    a_name##tsd_tls = a_initializer;					\
 a_attr __thread bool JEMALLOC_TLS_MODEL					\
-    a_name##_initialized = false;					\
+    a_name##tsd_initialized = false;					\
-a_attr bool		a_name##_booted = false;
+a_attr bool		a_name##tsd_booted = false;
 #elif (defined(JEMALLOC_TLS))
 #define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
 a_attr __thread a_type JEMALLOC_TLS_MODEL				\
-    a_name##_tls = a_initializer;					\
+    a_name##tsd_tls = a_initializer;					\
-a_attr pthread_key_t	a_name##_tsd;					\
+a_attr pthread_key_t	a_name##tsd_tsd;				\
-a_attr bool		a_name##_booted = false;
+a_attr bool		a_name##tsd_booted = false;
 #elif (defined(_WIN32))
 #define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
-a_attr DWORD		a_name##_tsd;					\
+a_attr DWORD		a_name##tsd_tsd;				\
-a_attr bool		a_name##_booted = false;
+a_attr bool		a_name##tsd_booted = false;
 #else
 #define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
-a_attr pthread_key_t	a_name##_tsd;					\
+a_attr pthread_key_t	a_name##tsd_tsd;				\
-a_attr tsd_init_head_t	a_name##_tsd_init_head = {			\
+a_attr tsd_init_head_t	a_name##tsd_init_head = {			\
 	ql_head_initializer(blocks),					\
 	MALLOC_MUTEX_INITIALIZER					\
 };									\
-a_attr bool		a_name##_booted = false;
+a_attr bool		a_name##tsd_booted = false;
 #endif
 
 /* malloc_tsd_funcs(). */
@@ -125,75 +132,76 @@ a_attr bool		a_name##_booted = false;
     a_cleanup)								\
 /* Initialization/cleanup. */						\
 a_attr bool								\
-a_name##_tsd_cleanup_wrapper(void)					\
+a_name##tsd_cleanup_wrapper(void)					\
 {									\
 									\
-	if (a_name##_initialized) {					\
+	if (a_name##tsd_initialized) {					\
-		a_name##_initialized = false;				\
+		a_name##tsd_initialized = false;			\
-		a_cleanup(&a_name##_tls);				\
+		a_cleanup(&a_name##tsd_tls);				\
 	}								\
-	return (a_name##_initialized);					\
+	return (a_name##tsd_initialized);				\
 }									\
 a_attr bool								\
-a_name##_tsd_boot(void)							\
+a_name##tsd_boot(void)							\
 {									\
 									\
 	if (a_cleanup != malloc_tsd_no_cleanup) {			\
 		malloc_tsd_cleanup_register(				\
-		    &a_name##_tsd_cleanup_wrapper);			\
+		    &a_name##tsd_cleanup_wrapper);			\
 	}								\
-	a_name##_booted = true;						\
+	a_name##tsd_booted = true;					\
 	return (false);							\
 }									\
 /* Get/set. */								\
 a_attr a_type *								\
-a_name##_tsd_get(void)							\
+a_name##tsd_get(void)							\
 {									\
 									\
-	assert(a_name##_booted);					\
+	assert(a_name##tsd_booted);					\
-	return (&a_name##_tls);						\
+	return (&a_name##tsd_tls);					\
 }									\
 a_attr void								\
-a_name##_tsd_set(a_type *val)						\
+a_name##tsd_set(a_type *val)						\
 {									\
 									\
-	assert(a_name##_booted);					\
+	assert(a_name##tsd_booted);					\
-	a_name##_tls = (*val);						\
+	a_name##tsd_tls = (*val);					\
 	if (a_cleanup != malloc_tsd_no_cleanup)				\
-		a_name##_initialized = true;				\
+		a_name##tsd_initialized = true;				\
 }
 #elif (defined(JEMALLOC_TLS))
 #define	malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,		\
     a_cleanup)								\
 /* Initialization/cleanup. */						\
 a_attr bool								\
-a_name##_tsd_boot(void)							\
+a_name##tsd_boot(void)							\
 {									\
 									\
 	if (a_cleanup != malloc_tsd_no_cleanup) {			\
-		if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0)	\
+		if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) !=	\
+		    0)							\
 			return (true);					\
 	}								\
-	a_name##_booted = true;						\
+	a_name##tsd_booted = true;					\
 	return (false);							\
 }									\
 /* Get/set. */								\
 a_attr a_type *								\
-a_name##_tsd_get(void)							\
+a_name##tsd_get(void)							\
 {									\
 									\
-	assert(a_name##_booted);					\
+	assert(a_name##tsd_booted);					\
-	return (&a_name##_tls);						\
+	return (&a_name##tsd_tls);					\
 }									\
 a_attr void								\
-a_name##_tsd_set(a_type *val)						\
+a_name##tsd_set(a_type *val)						\
 {									\
 									\
-	assert(a_name##_booted);					\
+	assert(a_name##tsd_booted);					\
-	a_name##_tls = (*val);						\
+	a_name##tsd_tls = (*val);					\
 	if (a_cleanup != malloc_tsd_no_cleanup) {			\
-		if (pthread_setspecific(a_name##_tsd,			\
+		if (pthread_setspecific(a_name##tsd_tsd,		\
-		    (void *)(&a_name##_tls))) {				\
+		    (void *)(&a_name##tsd_tls))) {			\
 			malloc_write("<jemalloc>: Error"		\
 			    " setting TSD for "#a_name"\n");		\
 			if (opt_abort)					\
@@ -208,23 +216,20 @@ a_name##_tsd_set(a_type *val)						\
 typedef struct {							\
 	bool	initialized;						\
 	a_type	val;							\
-} a_name##_tsd_wrapper_t;						\
+} a_name##tsd_wrapper_t;						\
 /* Initialization/cleanup. */						\
 a_attr bool								\
-a_name##_tsd_cleanup_wrapper(void)					\
+a_name##tsd_cleanup_wrapper(void)					\
 {									\
-	a_name##_tsd_wrapper_t *wrapper;				\
+	a_name##tsd_wrapper_t *wrapper;					\
 									\
-	wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd);	\
+	wrapper = (a_name##tsd_wrapper_t *)TlsGetValue(a_name##tsd_tsd);\
 	if (wrapper == NULL)						\
 		return (false);						\
 	if (a_cleanup != malloc_tsd_no_cleanup &&			\
 	    wrapper->initialized) {					\
-		a_type val = wrapper->val;				\
-		a_type tsd_static_data = a_initializer;			\
 		wrapper->initialized = false;				\
-		wrapper->val = tsd_static_data;				\
+		a_cleanup(&wrapper->val);				\
-		a_cleanup(&val);					\
 		if (wrapper->initialized) {				\
 			/* Trigger another cleanup round. */		\
 			return (true);					\
@@ -234,39 +239,38 @@ a_name##_tsd_cleanup_wrapper(void)					\
 	return (false);							\
 }									\
 a_attr bool								\
-a_name##_tsd_boot(void)							\
+a_name##tsd_boot(void)							\
 {									\
 									\
-	a_name##_tsd = TlsAlloc();					\
+	a_name##tsd_tsd = TlsAlloc();					\
-	if (a_name##_tsd == TLS_OUT_OF_INDEXES)				\
+	if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES)			\
 		return (true);						\
 	if (a_cleanup != malloc_tsd_no_cleanup) {			\
 		malloc_tsd_cleanup_register(				\
-		    &a_name##_tsd_cleanup_wrapper);			\
+		    &a_name##tsd_cleanup_wrapper);			\
 	}								\
-	a_name##_booted = true;						\
+	a_name##tsd_booted = true;					\
 	return (false);							\
 }									\
 /* Get/set. */								\
-a_attr a_name##_tsd_wrapper_t *						\
+a_attr a_name##tsd_wrapper_t *						\
-a_name##_tsd_get_wrapper(void)						\
+a_name##tsd_get_wrapper(void)						\
 {									\
-	a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)	\
+	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
-	    TlsGetValue(a_name##_tsd);					\
+	    TlsGetValue(a_name##tsd_tsd);				\
 									\
-	if (wrapper == NULL) {						\
+	if (unlikely(wrapper == NULL)) {				\
-		wrapper = (a_name##_tsd_wrapper_t *)			\
+		wrapper = (a_name##tsd_wrapper_t *)			\
-		    malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t));	\
+		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
 		if (wrapper == NULL) {					\
 			malloc_write("<jemalloc>: Error allocating"	\
 			    " TSD for "#a_name"\n");			\
 			abort();					\
 		} else {						\
-			static a_type tsd_static_data = a_initializer;	\
 			wrapper->initialized = false;			\
-			wrapper->val = tsd_static_data;			\
+			wrapper->val = a_initializer;			\
 		}							\
-		if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) {	\
+		if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) {	\
 			malloc_write("<jemalloc>: Error setting"	\
 			    " TSD for "#a_name"\n");			\
 			abort();					\
@@ -275,21 +279,21 @@ a_name##_tsd_get_wrapper(void)						\
 	return (wrapper);						\
 }									\
 a_attr a_type *								\
-a_name##_tsd_get(void)							\
+a_name##tsd_get(void)							\
 {									\
-	a_name##_tsd_wrapper_t *wrapper;				\
+	a_name##tsd_wrapper_t *wrapper;					\
 									\
-	assert(a_name##_booted);					\
+	assert(a_name##tsd_booted);					\
-	wrapper = a_name##_tsd_get_wrapper();				\
+	wrapper = a_name##tsd_get_wrapper();				\
 	return (&wrapper->val);						\
 }									\
 a_attr void								\
-a_name##_tsd_set(a_type *val)						\
+a_name##tsd_set(a_type *val)						\
 {									\
-	a_name##_tsd_wrapper_t *wrapper;				\
+	a_name##tsd_wrapper_t *wrapper;					\
 									\
-	assert(a_name##_booted);					\
+	assert(a_name##tsd_booted);					\
-	wrapper = a_name##_tsd_get_wrapper();				\
+	wrapper = a_name##tsd_get_wrapper();				\
 	wrapper->val = *(val);						\
 	if (a_cleanup != malloc_tsd_no_cleanup)				\
 		wrapper->initialized = true;				\
@@ -301,12 +305,12 @@ a_name##_tsd_set(a_type *val)						\
 typedef struct {							\
 	bool	initialized;						\
 	a_type	val;							\
-} a_name##_tsd_wrapper_t;						\
+} a_name##tsd_wrapper_t;						\
 /* Initialization/cleanup. */						\
 a_attr void								\
-a_name##_tsd_cleanup_wrapper(void *arg)					\
+a_name##tsd_cleanup_wrapper(void *arg)					\
 {									\
-	a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\
+	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg;	\
 									\
 	if (a_cleanup != malloc_tsd_no_cleanup &&			\
 	    wrapper->initialized) {					\
@@ -314,7 +318,7 @@ a_name##_tsd_cleanup_wrapper(void *arg)					\
 		a_cleanup(&wrapper->val);				\
 		if (wrapper->initialized) {				\
 			/* Trigger another cleanup round. */		\
-			if (pthread_setspecific(a_name##_tsd,		\
+			if (pthread_setspecific(a_name##tsd_tsd,	\
 			    (void *)wrapper)) {				\
 				malloc_write("<jemalloc>: Error"	\
 				    " setting TSD for "#a_name"\n");	\
@@ -327,66 +331,65 @@ a_name##_tsd_cleanup_wrapper(void *arg)					\
 	malloc_tsd_dalloc(wrapper);					\
 }									\
 a_attr bool								\
-a_name##_tsd_boot(void)							\
+a_name##tsd_boot(void)							\
 {									\
 									\
-	if (pthread_key_create(&a_name##_tsd,				\
+	if (pthread_key_create(&a_name##tsd_tsd,			\
-	    a_name##_tsd_cleanup_wrapper) != 0)				\
+	    a_name##tsd_cleanup_wrapper) != 0)				\
 		return (true);						\
-	a_name##_booted = true;						\
+	a_name##tsd_booted = true;					\
 	return (false);							\
 }									\
 /* Get/set. */								\
-a_attr a_name##_tsd_wrapper_t *						\
+a_attr a_name##tsd_wrapper_t *						\
-a_name##_tsd_get_wrapper(void)						\
+a_name##tsd_get_wrapper(void)						\
 {									\
-	a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)	\
+	a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)	\
-	    pthread_getspecific(a_name##_tsd);				\
+	    pthread_getspecific(a_name##tsd_tsd);			\
 									\
-	if (wrapper == NULL) {						\
+	if (unlikely(wrapper == NULL)) {				\
 		tsd_init_block_t block;					\
 		wrapper = tsd_init_check_recursion(			\
-		    &a_name##_tsd_init_head, &block);			\
+		    &a_name##tsd_init_head, &block);			\
 		if (wrapper)						\
 		    return (wrapper);					\
-		wrapper = (a_name##_tsd_wrapper_t *)			\
+		wrapper = (a_name##tsd_wrapper_t *)			\
-		    malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t));	\
+		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
 		block.data = wrapper;					\
 		if (wrapper == NULL) {					\
 			malloc_write("<jemalloc>: Error allocating"	\
 			    " TSD for "#a_name"\n");			\
 			abort();					\
 		} else {						\
-			static a_type tsd_static_data = a_initializer;	\
 			wrapper->initialized = false;			\
-			wrapper->val = tsd_static_data;			\
+			wrapper->val = a_initializer;			\
 		}							\
-		if (pthread_setspecific(a_name##_tsd,			\
+		if (pthread_setspecific(a_name##tsd_tsd,		\
 		    (void *)wrapper)) {					\
 			malloc_write("<jemalloc>: Error setting"	\
 			    " TSD for "#a_name"\n");			\
 			abort();					\
 		}							\
-		tsd_init_finish(&a_name##_tsd_init_head, &block);	\
+		tsd_init_finish(&a_name##tsd_init_head, &block);	\
 	}								\
 	return (wrapper);						\
 }									\
 a_attr a_type *								\
-a_name##_tsd_get(void)							\
+a_name##tsd_get(void)							\
 {									\
-	a_name##_tsd_wrapper_t *wrapper;				\
+	a_name##tsd_wrapper_t *wrapper;					\
 									\
-	assert(a_name##_booted);					\
+	assert(a_name##tsd_booted);					\
-	wrapper = a_name##_tsd_get_wrapper();				\
+	wrapper = a_name##tsd_get_wrapper();				\
 	return (&wrapper->val);						\
 }									\
 a_attr void								\
-a_name##_tsd_set(a_type *val)						\
+a_name##tsd_set(a_type *val)						\
 {									\
-	a_name##_tsd_wrapper_t *wrapper;				\
+	a_name##tsd_wrapper_t *wrapper;					\
 									\
-	assert(a_name##_booted);					\
+	assert(a_name##tsd_booted);					\
-	wrapper = a_name##_tsd_get_wrapper();				\
+	wrapper = a_name##tsd_get_wrapper();				\
 	wrapper->val = *(val);						\
 	if (a_cleanup != malloc_tsd_no_cleanup)				\
 		wrapper->initialized = true;				\
@@ -410,25 +413,123 @@ struct tsd_init_head_s {
 };
 #endif
 
+#define	MALLOC_TSD							\
+/*  O(name,			type) */				\
+    O(tcache,			tcache_t *)				\
+    O(thread_allocated,		uint64_t)				\
+    O(thread_deallocated,	uint64_t)				\
+    O(prof_tdata,		prof_tdata_t *)				\
+    O(arena,			arena_t *)				\
+    O(tcache_enabled,		tcache_enabled_t)			\
+    O(quarantine,		quarantine_t *)				\
+
+#define	TSD_INITIALIZER {						\
+    tsd_state_uninitialized,						\
+    NULL,								\
+    0,									\
+    0,									\
+    NULL,								\
+    NULL,								\
+    tcache_enabled_default,						\
+    NULL								\
+}
+
+struct tsd_s {
+	tsd_state_t	state;
+#define	O(n, t)								\
+	t		n;
+MALLOC_TSD
+#undef O
+};
+
+static const tsd_t tsd_initializer = TSD_INITIALIZER;
+
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
 void	*malloc_tsd_malloc(size_t size);
 void	malloc_tsd_dalloc(void *wrapper);
-void	malloc_tsd_no_cleanup(void *);
+void	malloc_tsd_no_cleanup(void *arg);
 void	malloc_tsd_cleanup_register(bool (*f)(void));
-void	malloc_tsd_boot(void);
+bool	malloc_tsd_boot(void);
 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
     !defined(_WIN32))
 void	*tsd_init_check_recursion(tsd_init_head_t *head,
     tsd_init_block_t *block);
 void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
 #endif
+void	tsd_cleanup(void *arg);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
+#ifndef JEMALLOC_ENABLE_INLINE
+malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
+
+tsd_t	*tsd_tryget(void);
+#define	O(n, t)								\
+t	*tsd_##n##p_get(tsd_t *tsd);					\
+t	tsd_##n##_get(tsd_t *tsd);					\
+void	tsd_##n##_set(tsd_t *tsd, t n);
+MALLOC_TSD
+#undef O
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
+malloc_tsd_externs(, tsd_t)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
+
+JEMALLOC_INLINE tsd_t *
+tsd_tryget(void)
+{
+	tsd_t *tsd;
+
+	tsd = tsd_get();
+	if (unlikely(tsd == NULL))
+		return (NULL);
+
+	if (likely(tsd->state == tsd_state_nominal))
+		return (tsd);
+	else if (tsd->state == tsd_state_uninitialized) {
+		tsd->state = tsd_state_nominal;
+		tsd_set(tsd);
+		return (tsd);
+	} else if (tsd->state == tsd_state_purgatory) {
+		tsd->state = tsd_state_reincarnated;
+		tsd_set(tsd);
+		return (NULL);
+	} else {
+		assert(tsd->state == tsd_state_reincarnated);
+		return (NULL);
+	}
+}
+
+#define	O(n, t)								\
+JEMALLOC_INLINE t *							\
+tsd_##n##p_get(tsd_t *tsd)						\
+{									\
+									\
+	return (&tsd->n);						\
+}									\
+									\
+JEMALLOC_INLINE t							\
+tsd_##n##_get(tsd_t *tsd)						\
+{									\
+									\
+	return (*tsd_##n##p_get(tsd));					\
+}									\
+									\
+JEMALLOC_INLINE void							\
+tsd_##n##_set(tsd_t *tsd, t n)						\
+{									\
+									\
+	tsd->n = n;							\
+}
+MALLOC_TSD
+#undef O
+#endif
+
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/

src/arena.c

@@ -2058,7 +2058,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
 }
 
 void *
-arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
+arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
     size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
     bool try_tcache_dalloc)
 {
@@ -2078,9 +2078,12 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
 		size_t usize = sa2u(size + extra, alignment);
 		if (usize == 0)
 			return (NULL);
-		ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
+		ret = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc,
-	} else
+		    arena);
-		ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc);
+	} else {
+		ret = arena_malloc(tsd, arena, size + extra, zero,
+		    try_tcache_alloc);
+	}
 
 	if (ret == NULL) {
 		if (extra == 0)
@@ -2090,10 +2093,12 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
 			size_t usize = sa2u(size, alignment);
 			if (usize == 0)
 				return (NULL);
-			ret = ipalloct(usize, alignment, zero, try_tcache_alloc,
+			ret = ipalloct(tsd, usize, alignment, zero,
-			    arena);
+			    try_tcache_alloc, arena);
-		} else
+		} else {
-			ret = arena_malloc(arena, size, zero, try_tcache_alloc);
+			ret = arena_malloc(tsd, arena, size, zero,
+			    try_tcache_alloc);
+		}
 
 		if (ret == NULL)
 			return (NULL);
@@ -2108,7 +2113,7 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
 	copysize = (size < oldsize) ? size : oldsize;
 	JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize);
 	memcpy(ret, ptr, copysize);
-	iqalloc(ptr, try_tcache_dalloc);
+	iqalloc(tsd, ptr, try_tcache_dalloc);
 	return (ret);
 }
 

src/ckh.c

@@ -40,8 +40,8 @@
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static bool	ckh_grow(ckh_t *ckh);
+static bool	ckh_grow(tsd_t *tsd, ckh_t *ckh);
-static void	ckh_shrink(ckh_t *ckh);
+static void	ckh_shrink(tsd_t *tsd, ckh_t *ckh);
 
 /******************************************************************************/
 
@@ -243,7 +243,7 @@ ckh_rebuild(ckh_t *ckh, ckhc_t *aTab)
 }
 
 static bool
-ckh_grow(ckh_t *ckh)
+ckh_grow(tsd_t *tsd, ckh_t *ckh)
 {
 	bool ret;
 	ckhc_t *tab, *ttab;
@@ -270,7 +270,7 @@ ckh_grow(ckh_t *ckh)
 			ret = true;
 			goto label_return;
 		}
-		tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
+		tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true);
 		if (tab == NULL) {
 			ret = true;
 			goto label_return;
@@ -282,12 +282,12 @@ ckh_grow(ckh_t *ckh)
 		ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
 
 		if (ckh_rebuild(ckh, tab) == false) {
-			idalloc(tab);
+			idalloc(tsd, tab);
 			break;
 		}
 
 		/* Rebuilding failed, so back out partially rebuilt table. */
-		idalloc(ckh->tab);
+		idalloc(tsd, ckh->tab);
 		ckh->tab = tab;
 		ckh->lg_curbuckets = lg_prevbuckets;
 	}
@@ -298,7 +298,7 @@ label_return:
 }
 
 static void
-ckh_shrink(ckh_t *ckh)
+ckh_shrink(tsd_t *tsd, ckh_t *ckh)
 {
 	ckhc_t *tab, *ttab;
 	size_t lg_curcells, usize;
@@ -313,7 +313,7 @@ ckh_shrink(ckh_t *ckh)
 	usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
 	if (usize == 0)
 		return;
-	tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
+	tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true);
 	if (tab == NULL) {
 		/*
 		 * An OOM error isn't worth propagating, since it doesn't
@@ -328,7 +328,7 @@ ckh_shrink(ckh_t *ckh)
 	ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
 
 	if (ckh_rebuild(ckh, tab) == false) {
-		idalloc(tab);
+		idalloc(tsd, tab);
 #ifdef CKH_COUNT
 		ckh->nshrinks++;
 #endif
@@ -336,7 +336,7 @@ ckh_shrink(ckh_t *ckh)
 	}
 
 	/* Rebuilding failed, so back out partially rebuilt table. */
-	idalloc(ckh->tab);
+	idalloc(tsd, ckh->tab);
 	ckh->tab = tab;
 	ckh->lg_curbuckets = lg_prevbuckets;
 #ifdef CKH_COUNT
@@ -345,7 +345,8 @@ ckh_shrink(ckh_t *ckh)
 }
 
 bool
-ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
+ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
+    ckh_keycomp_t *keycomp)
 {
 	bool ret;
 	size_t mincells, usize;
@@ -388,7 +389,7 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
 		ret = true;
 		goto label_return;
 	}
-	ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
+	ckh->tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true);
 	if (ckh->tab == NULL) {
 		ret = true;
 		goto label_return;
@@ -400,7 +401,7 @@ label_return:
 }
 
 void
-ckh_delete(ckh_t *ckh)
+ckh_delete(tsd_t *tsd, ckh_t *ckh)
 {
 
 	assert(ckh != NULL);
@@ -417,7 +418,7 @@ ckh_delete(ckh_t *ckh)
 	    (unsigned long long)ckh->nrelocs);
 #endif
 
-	idalloc(ckh->tab);
+	idalloc(tsd, ckh->tab);
 	if (config_debug)
 		memset(ckh, 0x5a, sizeof(ckh_t));
 }
@@ -452,7 +453,7 @@ ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data)
 }
 
 bool
-ckh_insert(ckh_t *ckh, const void *key, const void *data)
+ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data)
 {
 	bool ret;
 
@@ -464,7 +465,7 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data)
 #endif
 
 	while (ckh_try_insert(ckh, &key, &data)) {
-		if (ckh_grow(ckh)) {
+		if (ckh_grow(tsd, ckh)) {
 			ret = true;
 			goto label_return;
 		}
@@ -476,7 +477,8 @@ label_return:
 }
 
 bool
-ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data)
+ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
+    void **data)
 {
 	size_t cell;
 
@@ -497,7 +499,7 @@ ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data)
 		    + LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets
 		    > ckh->lg_minbuckets) {
 			/* Ignore error due to OOM. */
-			ckh_shrink(ckh);
+			ckh_shrink(tsd, ckh);
 		}
 
 		return (false);

src/ctl.c

@@ -565,18 +565,23 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
 static bool
 ctl_grow(void)
 {
+	tsd_t *tsd;
 	ctl_arena_stats_t *astats;
 	arena_t **tarenas;
 
+	tsd = tsd_tryget();
+	if (tsd == NULL)
+		return (true);
+
 	/* Allocate extended arena stats and arenas arrays. */
-	astats = (ctl_arena_stats_t *)imalloc((ctl_stats.narenas + 2) *
+	astats = (ctl_arena_stats_t *)imalloc(tsd, (ctl_stats.narenas + 2) *
 	    sizeof(ctl_arena_stats_t));
 	if (astats == NULL)
 		return (true);
-	tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) *
+	tarenas = (arena_t **)imalloc(tsd, (ctl_stats.narenas + 1) *
 	    sizeof(arena_t *));
 	if (tarenas == NULL) {
-		idalloc(astats);
+		idalloc(tsd, astats);
 		return (true);
 	}
 
@@ -585,8 +590,8 @@ ctl_grow(void)
 	    sizeof(ctl_arena_stats_t));
 	memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
 	if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) {
-		idalloc(tarenas);
+		idalloc(tsd, tarenas);
-		idalloc(astats);
+		idalloc(tsd, astats);
 		return (true);
 	}
 	/* Swap merged stats to their new location. */
@@ -623,7 +628,7 @@ ctl_grow(void)
 		 * base_alloc()).
 		 */
 		if (ctl_stats.narenas != narenas_auto)
-			idalloc(arenas_old);
+			idalloc(tsd, arenas_old);
 	}
 	ctl_stats.arenas = astats;
 	ctl_stats.narenas++;
@@ -1105,6 +1110,31 @@ label_return:								\
 	return (ret);							\
 }
 
+#define	CTL_TSD_RO_NL_CGEN(c, n, m, t)					\
+static int								\
+n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,	\
+    void *newp, size_t newlen)						\
+{									\
+	int ret;							\
+	t oldval;							\
+	tsd_t *tsd;							\
+									\
+	if ((c) == false)						\
+		return (ENOENT);					\
+	READONLY();							\
+	tsd = tsd_tryget();						\
+	if (tsd == NULL) {						\
+		ret = EAGAIN;						\
+		goto label_return;					\
+	}								\
+	oldval = (m(tsd));						\
+	READ(oldval, t);						\
+									\
+	ret = 0;							\
+label_return:								\
+	return (ret);							\
+}
+
 #define	CTL_RO_BOOL_CONFIG_GEN(n)					\
 static int								\
 n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,	\
@@ -1194,10 +1224,15 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
     void *newp, size_t newlen)
 {
 	int ret;
+	tsd_t *tsd;
 	unsigned newind, oldind;
 
+	tsd = tsd_tryget();
+	if (tsd == NULL)
+		return (EAGAIN);
+
 	malloc_mutex_lock(&ctl_mtx);
-	newind = oldind = choose_arena(NULL)->ind;
+	newind = oldind = choose_arena(tsd, NULL)->ind;
 	WRITE(newind, unsigned);
 	READ(oldind, unsigned);
 	if (newind != oldind) {
@@ -1224,14 +1259,14 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 
 		/* Set new arena association. */
 		if (config_tcache) {
-			tcache_t *tcache;
+			tcache_t *tcache = tsd_tcache_get(tsd);
-			if ((uintptr_t)(tcache = *tcache_tsd_get()) >
+			if (tcache != NULL) {
-			    (uintptr_t)TCACHE_STATE_MAX) {
 				tcache_arena_dissociate(tcache);
 				tcache_arena_associate(tcache, arena);
 			}
 		}
-		arenas_tsd_set(&arena);
+
+		tsd_arena_set(tsd, arena);
 	}
 
 	ret = 0;
@@ -1240,14 +1275,14 @@ label_return:
 	return (ret);
 }
 
-CTL_RO_NL_CGEN(config_stats, thread_allocated,
+CTL_TSD_RO_NL_CGEN(config_stats, thread_allocated, tsd_thread_allocated_get,
-    thread_allocated_tsd_get()->allocated, uint64_t)
+    uint64_t)
-CTL_RO_NL_CGEN(config_stats, thread_allocatedp,
+CTL_TSD_RO_NL_CGEN(config_stats, thread_allocatedp, tsd_thread_allocatedp_get,
-    &thread_allocated_tsd_get()->allocated, uint64_t *)
+    uint64_t *)
-CTL_RO_NL_CGEN(config_stats, thread_deallocated,
+CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocated, tsd_thread_deallocated_get,
-    thread_allocated_tsd_get()->deallocated, uint64_t)
+    uint64_t)
-CTL_RO_NL_CGEN(config_stats, thread_deallocatedp,
+CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp,
-    &thread_allocated_tsd_get()->deallocated, uint64_t *)
+    tsd_thread_deallocatedp_get, uint64_t *)
 
 static int
 thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
@@ -1305,11 +1340,20 @@ thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp,
 
 	oldname = prof_thread_name_get();
 	if (newp != NULL) {
+		tsd_t *tsd;
+
 		if (newlen != sizeof(const char *)) {
 			ret = EINVAL;
 			goto label_return;
 		}
-		if (prof_thread_name_set(*(const char **)newp)) {
+
+		tsd = tsd_tryget();
+		if (tsd == NULL) {
+			ret = EAGAIN;
+			goto label_return;
+		}
+
+		if (prof_thread_name_set(tsd, *(const char **)newp)) {
 			ret = EAGAIN;
 			goto label_return;
 		}
@@ -1675,6 +1719,7 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 {
 	int ret;
 	size_t lg_sample = lg_prof_sample;
+	tsd_t *tsd;
 
 	if (config_prof == false)
 		return (ENOENT);
@@ -1684,7 +1729,13 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 	if (lg_sample >= (sizeof(uint64_t) << 3))
 		lg_sample = (sizeof(uint64_t) << 3) - 1;
 
-	prof_reset(lg_sample);
+	tsd = tsd_tryget();
+	if (tsd == NULL) {
+		ret = EAGAIN;
+		goto label_return;
+	}
+
+	prof_reset(tsd, lg_sample);
 
 	ret = 0;
 label_return:

src/huge.c

@@ -13,14 +13,15 @@ static malloc_mutex_t	huge_mtx;
 static extent_tree_t	huge;
 
 void *
-huge_malloc(arena_t *arena, size_t size, bool zero)
+huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero)
 {
 
-	return (huge_palloc(arena, size, chunksize, zero));
+	return (huge_palloc(tsd, arena, size, chunksize, zero));
 }
 
 void *
-huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
+huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
+    bool zero)
 {
 	void *ret;
 	size_t csize;
@@ -45,7 +46,7 @@ huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
 	 * it is possible to make correct junk/zero fill decisions below.
 	 */
 	is_zeroed = zero;
-	arena = choose_arena(arena);
+	arena = choose_arena(tsd, arena);
 	ret = arena_chunk_alloc_huge(arena, csize, alignment, &is_zeroed);
 	if (ret == NULL) {
 		base_node_dalloc(node);
@@ -90,7 +91,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
 }
 
 void *
-huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
+huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
     size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc)
 {
 	void *ret;
@@ -106,18 +107,18 @@ huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
 	 * space and copying.
 	 */
 	if (alignment > chunksize)
-		ret = huge_palloc(arena, size + extra, alignment, zero);
+		ret = huge_palloc(tsd, arena, size + extra, alignment, zero);
 	else
-		ret = huge_malloc(arena, size + extra, zero);
+		ret = huge_malloc(tsd, arena, size + extra, zero);
 
 	if (ret == NULL) {
 		if (extra == 0)
 			return (NULL);
 		/* Try again, this time without extra. */
 		if (alignment > chunksize)
-			ret = huge_palloc(arena, size, alignment, zero);
+			ret = huge_palloc(tsd, arena, size, alignment, zero);
 		else
-			ret = huge_malloc(arena, size, zero);
+			ret = huge_malloc(tsd, arena, size, zero);
 
 		if (ret == NULL)
 			return (NULL);
@@ -129,7 +130,7 @@ huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
 	 */
 	copysize = (size < oldsize) ? size : oldsize;
 	memcpy(ret, ptr, copysize);
-	iqalloc(ptr, try_tcache_dalloc);
+	iqalloc(tsd, ptr, try_tcache_dalloc);
 	return (ret);
 }
 

src/jemalloc.c

@@ -5,8 +5,6 @@
 /* Data. */
 
 malloc_tsd_data(, arenas, arena_t *, NULL)
-malloc_tsd_data(, thread_allocated, thread_allocated_t,
-    THREAD_ALLOCATED_INITIALIZER)
 
 /* Runtime configuration options. */
 const char	*je_malloc_conf;
@@ -142,7 +140,7 @@ arenas_extend(unsigned ind)
 
 /* Slow path, called only by choose_arena(). */
 arena_t *
-choose_arena_hard(void)
+choose_arena_hard(tsd_t *tsd)
 {
 	arena_t *ret;
 
@@ -196,11 +194,32 @@ choose_arena_hard(void)
 		malloc_mutex_unlock(&arenas_lock);
 	}
 
-	arenas_tsd_set(&ret);
+	tsd_arena_set(tsd, ret);
 
 	return (ret);
 }
 
+void
+thread_allocated_cleanup(tsd_t *tsd)
+{
+
+	/* Do nothing. */
+}
+
+void
+thread_deallocated_cleanup(tsd_t *tsd)
+{
+
+	/* Do nothing. */
+}
+
+void
+arena_cleanup(tsd_t *tsd)
+{
+
+	/* Do nothing. */
+}
+
 static void
 stats_print_atexit(void)
 {
@@ -691,7 +710,11 @@ malloc_init_hard(void)
 #endif
 	malloc_initializer = INITIALIZER;
 
-	malloc_tsd_boot();
+	if (malloc_tsd_boot()) {
+		malloc_mutex_unlock(&init_lock);
+		return (true);
+	}
+
 	if (config_prof)
 		prof_boot0();
 
@@ -726,7 +749,7 @@ malloc_init_hard(void)
 
 	arena_boot();
 
-	if (config_tcache && tcache_boot0()) {
+	if (config_tcache && tcache_boot()) {
 		malloc_mutex_unlock(&init_lock);
 		return (true);
 	}
@@ -759,27 +782,6 @@ malloc_init_hard(void)
 		return (true);
 	}
 
-	/* Initialize allocation counters before any allocations can occur. */
-	if (config_stats && thread_allocated_tsd_boot()) {
-		malloc_mutex_unlock(&init_lock);
-		return (true);
-	}
-
-	if (arenas_tsd_boot()) {
-		malloc_mutex_unlock(&init_lock);
-		return (true);
-	}
-
-	if (config_tcache && tcache_boot1()) {
-		malloc_mutex_unlock(&init_lock);
-		return (true);
-	}
-
-	if (config_fill && quarantine_boot()) {
-		malloc_mutex_unlock(&init_lock);
-		return (true);
-	}
-
 	if (config_prof && prof_boot2()) {
 		malloc_mutex_unlock(&init_lock);
 		return (true);
@@ -863,36 +865,36 @@ malloc_init_hard(void)
  */
 
 static void *
-imalloc_prof_sample(size_t usize, prof_tctx_t *tctx)
+imalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
 {
 	void *p;
 
 	if (tctx == NULL)
 		return (NULL);
 	if (usize <= SMALL_MAXCLASS) {
-		p = imalloc(LARGE_MINCLASS);
+		p = imalloc(tsd, LARGE_MINCLASS);
 		if (p == NULL)
 			return (NULL);
 		arena_prof_promoted(p, usize);
 	} else
-		p = imalloc(usize);
+		p = imalloc(tsd, usize);
 
 	return (p);
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-imalloc_prof(size_t usize)
+imalloc_prof(tsd_t *tsd, size_t usize)
 {
 	void *p;
 	prof_tctx_t *tctx;
 
-	tctx = prof_alloc_prep(usize, true);
+	tctx = prof_alloc_prep(tsd, usize, true);
 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
-		p = imalloc_prof_sample(usize, tctx);
+		p = imalloc_prof_sample(tsd, usize, tctx);
 	else
-		p = imalloc(usize);
+		p = imalloc(tsd, usize);
 	if (p == NULL) {
-		prof_alloc_rollback(tctx, true);
+		prof_alloc_rollback(tsd, tctx, true);
 		return (NULL);
 	}
 	prof_malloc(p, usize, tctx);
@@ -901,32 +903,33 @@ imalloc_prof(size_t usize)
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-imalloc_body(size_t size, size_t *usize)
+imalloc_body(size_t size, tsd_t **tsd, size_t *usize)
 {
 
-	if (unlikely(malloc_init()))
+	if (unlikely(malloc_init()) || unlikely((*tsd = tsd_tryget()) == NULL))
 		return (NULL);
 
 	if (config_prof && opt_prof) {
 		*usize = s2u(size);
-		return (imalloc_prof(*usize));
+		return (imalloc_prof(*tsd, *usize));
 	}
 
 	if (config_stats || (config_valgrind && unlikely(in_valgrind)))
 		*usize = s2u(size);
-	return (imalloc(size));
+	return (imalloc(*tsd, size));
 }
 
 void *
 je_malloc(size_t size)
 {
 	void *ret;
+	tsd_t *tsd;
 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
 
 	if (size == 0)
 		size = 1;
 
-	ret = imalloc_body(size, &usize);
+	ret = imalloc_body(size, &tsd, &usize);
 	if (unlikely(ret == NULL)) {
 		if (config_xmalloc && unlikely(opt_xmalloc)) {
 			malloc_write("<jemalloc>: Error in malloc(): "
@@ -937,7 +940,7 @@ je_malloc(size_t size)
 	}
 	if (config_stats && likely(ret != NULL)) {
 		assert(usize == isalloc(ret, config_prof));
-		thread_allocated_tsd_get()->allocated += usize;
+		*tsd_thread_allocatedp_get(tsd) += usize;
 	}
 	UTRACE(0, size, ret);
 	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
@@ -945,7 +948,8 @@ je_malloc(size_t size)
 }
 
 static void *
-imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx)
+imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
+    prof_tctx_t *tctx)
 {
 	void *p;
 
@@ -953,29 +957,29 @@ imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx)
 		return (NULL);
 	if (usize <= SMALL_MAXCLASS) {
 		assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS);
-		p = imalloc(LARGE_MINCLASS);
+		p = imalloc(tsd, LARGE_MINCLASS);
 		if (p == NULL)
 			return (NULL);
 		arena_prof_promoted(p, usize);
 	} else
-		p = ipalloc(usize, alignment, false);
+		p = ipalloc(tsd, usize, alignment, false);
 
 	return (p);
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-imemalign_prof(size_t alignment, size_t usize)
+imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
 {
 	void *p;
 	prof_tctx_t *tctx;
 
-	tctx = prof_alloc_prep(usize, true);
+	tctx = prof_alloc_prep(tsd, usize, true);
 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
-		p = imemalign_prof_sample(alignment, usize, tctx);
+		p = imemalign_prof_sample(tsd, alignment, usize, tctx);
 	else
-		p = ipalloc(usize, alignment, false);
+		p = ipalloc(tsd, usize, alignment, false);
 	if (p == NULL) {
-		prof_alloc_rollback(tctx, true);
+		prof_alloc_rollback(tsd, tctx, true);
 		return (NULL);
 	}
 	prof_malloc(p, usize, tctx);
@@ -988,12 +992,13 @@ static int
 imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
 {
 	int ret;
+	tsd_t *tsd;
 	size_t usize;
 	void *result;
 
 	assert(min_alignment != 0);
 
-	if (unlikely(malloc_init())) {
+	if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) {
 		result = NULL;
 		goto label_oom;
 	} else {
@@ -1020,9 +1025,9 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
 		}
 
 		if (config_prof && opt_prof)
-			result = imemalign_prof(alignment, usize);
+			result = imemalign_prof(tsd, alignment, usize);
 		else
-			result = ipalloc(usize, alignment, false);
+			result = ipalloc(tsd, usize, alignment, false);
 		if (unlikely(result == NULL))
 			goto label_oom;
 	}
@@ -1032,7 +1037,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
 label_return:
 	if (config_stats && likely(result != NULL)) {
 		assert(usize == isalloc(result, config_prof));
-		thread_allocated_tsd_get()->allocated += usize;
+		*tsd_thread_allocatedp_get(tsd) += usize;
 	}
 	UTRACE(0, size, result);
 	return (ret);
@@ -1072,36 +1077,36 @@ je_aligned_alloc(size_t alignment, size_t size)
 }
 
 static void *
-icalloc_prof_sample(size_t usize, prof_tctx_t *tctx)
+icalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
 {
 	void *p;
 
 	if (tctx == NULL)
 		return (NULL);
 	if (usize <= SMALL_MAXCLASS) {
-		p = icalloc(LARGE_MINCLASS);
+		p = icalloc(tsd, LARGE_MINCLASS);
 		if (p == NULL)
 			return (NULL);
 		arena_prof_promoted(p, usize);
 	} else
-		p = icalloc(usize);
+		p = icalloc(tsd, usize);
 
 	return (p);
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-icalloc_prof(size_t usize)
+icalloc_prof(tsd_t *tsd, size_t usize)
 {
 	void *p;
 	prof_tctx_t *tctx;
 
-	tctx = prof_alloc_prep(usize, true);
+	tctx = prof_alloc_prep(tsd, usize, true);
 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
-		p = icalloc_prof_sample(usize, tctx);
+		p = icalloc_prof_sample(tsd, usize, tctx);
 	else
-		p = icalloc(usize);
+		p = icalloc(tsd, usize);
 	if (p == NULL) {
-		prof_alloc_rollback(tctx, true);
+		prof_alloc_rollback(tsd, tctx, true);
 		return (NULL);
 	}
 	prof_malloc(p, usize, tctx);
@@ -1113,10 +1118,11 @@ void *
 je_calloc(size_t num, size_t size)
 {
 	void *ret;
+	tsd_t *tsd;
 	size_t num_size;
 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
 
-	if (unlikely(malloc_init())) {
+	if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) {
 		num_size = 0;
 		ret = NULL;
 		goto label_return;
@@ -1144,11 +1150,11 @@ je_calloc(size_t num, size_t size)
 
 	if (config_prof && opt_prof) {
 		usize = s2u(num_size);
-		ret = icalloc_prof(usize);
+		ret = icalloc_prof(tsd, usize);
 	} else {
 		if (config_stats || (config_valgrind && unlikely(in_valgrind)))
 			usize = s2u(num_size);
-		ret = icalloc(num_size);
+		ret = icalloc(tsd, num_size);
 	}
 
 label_return:
@@ -1162,7 +1168,7 @@ label_return:
 	}
 	if (config_stats && likely(ret != NULL)) {
 		assert(usize == isalloc(ret, config_prof));
-		thread_allocated_tsd_get()->allocated += usize;
+		*tsd_thread_allocatedp_get(tsd) += usize;
 	}
 	UTRACE(0, num_size, ret);
 	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
@@ -1170,44 +1176,44 @@ label_return:
 }
 
 static void *
-irealloc_prof_sample(void *oldptr, size_t usize, prof_tctx_t *tctx)
+irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t usize, prof_tctx_t *tctx)
 {
 	void *p;
 
 	if (tctx == NULL)
 		return (NULL);
 	if (usize <= SMALL_MAXCLASS) {
-		p = iralloc(oldptr, LARGE_MINCLASS, 0, false);
+		p = iralloc(tsd, oldptr, LARGE_MINCLASS, 0, false);
 		if (p == NULL)
 			return (NULL);
 		arena_prof_promoted(p, usize);
 	} else
-		p = iralloc(oldptr, usize, 0, false);
+		p = iralloc(tsd, oldptr, usize, 0, false);
 
 	return (p);
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-irealloc_prof(void *oldptr, size_t old_usize, size_t usize)
+irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize)
 {
 	void *p;
 	prof_tctx_t *old_tctx, *tctx;
 
 	old_tctx = prof_tctx_get(oldptr);
-	tctx = prof_alloc_prep(usize, true);
+	tctx = prof_alloc_prep(tsd, usize, true);
 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
-		p = irealloc_prof_sample(oldptr, usize, tctx);
+		p = irealloc_prof_sample(tsd, oldptr, usize, tctx);
 	else
-		p = iralloc(oldptr, usize, 0, false);
+		p = iralloc(tsd, oldptr, usize, 0, false);
 	if (p == NULL)
 		return (NULL);
-	prof_realloc(p, usize, tctx, true, old_usize, old_tctx);
+	prof_realloc(tsd, p, usize, tctx, true, old_usize, old_tctx);
 
 	return (p);
 }
 
 JEMALLOC_INLINE_C void
-ifree(void *ptr, bool try_tcache)
+ifree(tsd_t *tsd, void *ptr, bool try_tcache)
 {
 	size_t usize;
 	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
@@ -1217,19 +1223,19 @@ ifree(void *ptr, bool try_tcache)
 
 	if (config_prof && opt_prof) {
 		usize = isalloc(ptr, config_prof);
-		prof_free(ptr, usize);
+		prof_free(tsd, ptr, usize);
 	} else if (config_stats || config_valgrind)
 		usize = isalloc(ptr, config_prof);
-	if (config_stats)
+	if (config_stats && likely(tsd != NULL))
-		thread_allocated_tsd_get()->deallocated += usize;
+		*tsd_thread_deallocatedp_get(tsd) += usize;
 	if (config_valgrind && unlikely(in_valgrind))
 		rzsize = p2rz(ptr);
-	iqalloc(ptr, try_tcache);
+	iqalloc(tsd, ptr, try_tcache);
 	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
 }
 
 JEMALLOC_INLINE_C void
-isfree(void *ptr, size_t usize, bool try_tcache)
+isfree(tsd_t *tsd, void *ptr, size_t usize, bool try_tcache)
 {
 	UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
 
@@ -1237,12 +1243,12 @@ isfree(void *ptr, size_t usize, bool try_tcache)
 	assert(malloc_initialized || IS_INITIALIZER);
 
 	if (config_prof && opt_prof)
-		prof_free(ptr, usize);
+		prof_free(tsd, ptr, usize);
-	if (config_stats)
+	if (config_stats && likely(tsd != NULL))
-		thread_allocated_tsd_get()->deallocated += usize;
+		*tsd_thread_deallocatedp_get(tsd) += usize;
 	if (config_valgrind && unlikely(in_valgrind))
 		rzsize = p2rz(ptr);
-	isqalloc(ptr, usize, try_tcache);
+	isqalloc(tsd, ptr, usize, try_tcache);
 	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
 }
 
@@ -1250,6 +1256,7 @@ void *
 je_realloc(void *ptr, size_t size)
 {
 	void *ret;
+	tsd_t *tsd;
 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
 	size_t old_usize = 0;
 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
@@ -1258,7 +1265,8 @@ je_realloc(void *ptr, size_t size)
 		if (ptr != NULL) {
 			/* realloc(ptr, 0) is equivalent to free(ptr). */
 			UTRACE(ptr, 0, 0);
-			ifree(ptr, true);
+			tsd = tsd_tryget();
+			ifree(tsd, ptr, true);
 			return (NULL);
 		}
 		size = 1;
@@ -1268,24 +1276,29 @@ je_realloc(void *ptr, size_t size)
 		assert(malloc_initialized || IS_INITIALIZER);
 		malloc_thread_init();
 
+		if ((tsd = tsd_tryget()) != NULL) {
 			if ((config_prof && opt_prof) || config_stats ||
 			    (config_valgrind && unlikely(in_valgrind)))
 				old_usize = isalloc(ptr, config_prof);
-		if (config_valgrind && unlikely(in_valgrind))
+			if (config_valgrind && unlikely(in_valgrind)) {
-			old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize);
+				old_rzsize = config_prof ? p2rz(ptr) :
+				    u2rz(old_usize);
+			}
 
 			if (config_prof && opt_prof) {
 				usize = s2u(size);
-			ret = irealloc_prof(ptr, old_usize, usize);
+				ret = irealloc_prof(tsd, ptr, old_usize, usize);
 			} else {
 				if (config_stats || (config_valgrind &&
 				    unlikely(in_valgrind)))
 					usize = s2u(size);
-			ret = iralloc(ptr, size, 0, false);
+				ret = iralloc(tsd, ptr, size, 0, false);
 			}
+		} else
+			ret = NULL;
 	} else {
 		/* realloc(NULL, size) is equivalent to malloc(size). */
-		ret = imalloc_body(size, &usize);
+		ret = imalloc_body(size, &tsd, &usize);
 	}
 
 	if (unlikely(ret == NULL)) {
@@ -1297,11 +1310,11 @@ je_realloc(void *ptr, size_t size)
 		set_errno(ENOMEM);
 	}
 	if (config_stats && likely(ret != NULL)) {
-		thread_allocated_t *ta;
 		assert(usize == isalloc(ret, config_prof));
-		ta = thread_allocated_tsd_get();
+		if (tsd != NULL) {
-		ta->allocated += usize;
+			*tsd_thread_allocatedp_get(tsd) += usize;
-		ta->deallocated += old_usize;
+			*tsd_thread_deallocatedp_get(tsd) += old_usize;
+		}
 	}
 	UTRACE(ptr, size, ret);
 	JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize,
@@ -1315,7 +1328,7 @@ je_free(void *ptr)
 
 	UTRACE(ptr, 0, 0);
 	if (likely(ptr != NULL))
-		ifree(ptr, true);
+		ifree(tsd_tryget(), ptr, true);
 }
 
 /*
@@ -1425,50 +1438,52 @@ imallocx_flags_decode(size_t size, int flags, size_t *usize, size_t *alignment,
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-imallocx_flags(size_t usize, size_t alignment, bool zero, bool try_tcache,
+imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
-    arena_t *arena)
+    bool try_tcache, arena_t *arena)
 {
 
-	if (alignment != 0)
+	if (alignment != 0) {
-		return (ipalloct(usize, alignment, zero, try_tcache, arena));
+		return (ipalloct(tsd, usize, alignment, zero, try_tcache,
+		    arena));
+	}
 	if (zero)
-		return (icalloct(usize, try_tcache, arena));
+		return (icalloct(tsd, usize, try_tcache, arena));
-	return (imalloct(usize, try_tcache, arena));
+	return (imalloct(tsd, usize, try_tcache, arena));
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-imallocx_maybe_flags(size_t size, int flags, size_t usize, size_t alignment,
+imallocx_maybe_flags(tsd_t *tsd, size_t size, int flags, size_t usize,
-    bool zero, bool try_tcache, arena_t *arena)
+    size_t alignment, bool zero, bool try_tcache, arena_t *arena)
 {
 
 	if (likely(flags == 0))
-		return (imalloc(size));
+		return (imalloc(tsd, size));
-	return (imallocx_flags(usize, alignment, zero, try_tcache, arena));
+	return (imallocx_flags(tsd, usize, alignment, zero, try_tcache, arena));
 }
 
 static void *
-imallocx_prof_sample(size_t size, int flags, size_t usize, size_t alignment,
+imallocx_prof_sample(tsd_t *tsd, size_t size, int flags, size_t usize,
-    bool zero, bool try_tcache, arena_t *arena)
+    size_t alignment, bool zero, bool try_tcache, arena_t *arena)
 {
 	void *p;
 
 	if (usize <= SMALL_MAXCLASS) {
 		assert(((alignment == 0) ? s2u(LARGE_MINCLASS) :
 		    sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS);
-		p = imalloct(LARGE_MINCLASS, try_tcache, arena);
+		p = imalloct(tsd, LARGE_MINCLASS, try_tcache, arena);
 		if (p == NULL)
 			return (NULL);
 		arena_prof_promoted(p, usize);
 	} else {
-		p = imallocx_maybe_flags(size, flags, usize, alignment, zero,
+		p = imallocx_maybe_flags(tsd, size, flags, usize, alignment,
-		    try_tcache, arena);
+		    zero, try_tcache, arena);
 	}
 
 	return (p);
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-imallocx_prof(size_t size, int flags, size_t *usize)
+imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize)
 {
 	void *p;
 	size_t alignment;
@@ -1479,17 +1494,17 @@ imallocx_prof(size_t size, int flags, size_t *usize)
 
 	imallocx_flags_decode(size, flags, usize, &alignment, &zero,
 	    &try_tcache, &arena);
-	tctx = prof_alloc_prep(*usize, true);
+	tctx = prof_alloc_prep(tsd, *usize, true);
 	if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
-		p = imallocx_maybe_flags(size, flags, *usize, alignment, zero,
+		p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment,
-		    try_tcache, arena);
+		    zero, try_tcache, arena);
 	} else if ((uintptr_t)tctx > (uintptr_t)1U) {
-		p = imallocx_prof_sample(size, flags, *usize, alignment, zero,
+		p = imallocx_prof_sample(tsd, size, flags, *usize, alignment,
-		    try_tcache, arena);
+		    zero, try_tcache, arena);
 	} else
 		p = NULL;
 	if (unlikely(p == NULL)) {
-		prof_alloc_rollback(tctx, true);
+		prof_alloc_rollback(tsd, tctx, true);
 		return (NULL);
 	}
 	prof_malloc(p, *usize, tctx);
@@ -1498,7 +1513,7 @@ imallocx_prof(size_t size, int flags, size_t *usize)
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-imallocx_no_prof(size_t size, int flags, size_t *usize)
+imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize)
 {
 	size_t alignment;
 	bool zero;
@@ -1508,35 +1523,39 @@ imallocx_no_prof(size_t size, int flags, size_t *usize)
 	if (likely(flags == 0)) {
 		if (config_stats || (config_valgrind && unlikely(in_valgrind)))
 			*usize = s2u(size);
-		return (imalloc(size));
+		return (imalloc(tsd, size));
 	}
 
 	imallocx_flags_decode_hard(size, flags, usize, &alignment, &zero,
 	    &try_tcache, &arena);
-	return (imallocx_flags(*usize, alignment, zero, try_tcache, arena));
+	return (imallocx_flags(tsd, *usize, alignment, zero, try_tcache,
+	    arena));
 }
 
 void *
 je_mallocx(size_t size, int flags)
 {
+	tsd_t *tsd;
 	void *p;
 	size_t usize;
 
 	assert(size != 0);
 
-	if (unlikely(malloc_init()))
+	if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL))
 		goto label_oom;
 
 	if (config_prof && opt_prof)
-		p = imallocx_prof(size, flags, &usize);
+		p = imallocx_prof(tsd, size, flags, &usize);
 	else
-		p = imallocx_no_prof(size, flags, &usize);
+		p = imallocx_no_prof(tsd, size, flags, &usize);
 	if (unlikely(p == NULL))
 		goto label_oom;
 
 	if (config_stats) {
+		tsd_t *tsd = tsd_tryget();
 		assert(usize == isalloc(p, config_prof));
-		thread_allocated_tsd_get()->allocated += usize;
+		if (tsd != NULL)
+			*tsd_thread_allocatedp_get(tsd) += usize;
 	}
 	UTRACE(0, size, p);
 	JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags));
@@ -1551,47 +1570,47 @@ label_oom:
 }
 
 static void *
-irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize,
+irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t size, size_t alignment,
-    bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena,
+    size_t usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
-    prof_tctx_t *tctx)
+    arena_t *arena, prof_tctx_t *tctx)
 {
 	void *p;
 
 	if (tctx == NULL)
 		return (NULL);
 	if (usize <= SMALL_MAXCLASS) {
-		p = iralloct(oldptr, LARGE_MINCLASS, alignment, zero,
+		p = iralloct(tsd, oldptr, LARGE_MINCLASS, alignment, zero,
 		    try_tcache_alloc, try_tcache_dalloc, arena);
 		if (p == NULL)
 			return (NULL);
 		arena_prof_promoted(p, usize);
 	} else {
-		p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc,
+		p = iralloct(tsd, oldptr, size, alignment, zero,
-		    try_tcache_dalloc, arena);
+		    try_tcache_alloc, try_tcache_dalloc, arena);
 	}
 
 	return (p);
 }
 
 JEMALLOC_ALWAYS_INLINE_C void *
-irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
+irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size,
-    size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
+    size_t alignment, size_t *usize, bool zero, bool try_tcache_alloc,
-    arena_t *arena)
+    bool try_tcache_dalloc, arena_t *arena)
 {
 	void *p;
 	prof_tctx_t *old_tctx, *tctx;
 
 	old_tctx = prof_tctx_get(oldptr);
-	tctx = prof_alloc_prep(*usize, false);
+	tctx = prof_alloc_prep(tsd, *usize, false);
 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
-		p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero,
+		p = irallocx_prof_sample(tsd, oldptr, size, alignment, *usize,
-		    try_tcache_alloc, try_tcache_dalloc, arena, tctx);
+		    zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx);
 	} else {
-		p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc,
+		p = iralloct(tsd, oldptr, size, alignment, zero,
-		    try_tcache_dalloc, arena);
+		    try_tcache_alloc, try_tcache_dalloc, arena);
 	}
 	if (unlikely(p == NULL)) {
-		prof_alloc_rollback(tctx, false);
+		prof_alloc_rollback(tsd, tctx, false);
 		return (NULL);
 	}
 
@@ -1606,7 +1625,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
 		 */
 		*usize = isalloc(p, config_prof);
 	}
-	prof_realloc(p, *usize, tctx, false, old_usize, old_tctx);
+	prof_realloc(tsd, p, *usize, tctx, false, old_usize, old_tctx);
 
 	return (p);
 }
@@ -1615,6 +1634,7 @@ void *
 je_rallocx(void *ptr, size_t size, int flags)
 {
 	void *p;
+	tsd_t *tsd;
 	size_t usize;
 	UNUSED size_t old_usize JEMALLOC_CC_SILENCE_INIT(0);
 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
@@ -1628,6 +1648,9 @@ je_rallocx(void *ptr, size_t size, int flags)
 	assert(malloc_initialized || IS_INITIALIZER);
 	malloc_thread_init();
 
+	if (unlikely((tsd = tsd_tryget()) == NULL))
+		goto label_oom;
+
 	if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
 		unsigned arena_ind = MALLOCX_ARENA_GET(flags);
 		arena_chunk_t *chunk;
@@ -1651,12 +1674,12 @@ je_rallocx(void *ptr, size_t size, int flags)
 	if (config_prof && opt_prof) {
 		usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
 		assert(usize != 0);
-		p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero,
+		p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
-		    try_tcache_alloc, try_tcache_dalloc, arena);
+		    zero, try_tcache_alloc, try_tcache_dalloc, arena);
 		if (unlikely(p == NULL))
 			goto label_oom;
 	} else {
-		p = iralloct(ptr, size, alignment, zero, try_tcache_alloc,
+		p = iralloct(tsd, ptr, size, alignment, zero, try_tcache_alloc,
 		    try_tcache_dalloc, arena);
 		if (unlikely(p == NULL))
 			goto label_oom;
@@ -1665,10 +1688,8 @@ je_rallocx(void *ptr, size_t size, int flags)
 	}
 
 	if (config_stats) {
-		thread_allocated_t *ta;
+		*tsd_thread_allocatedp_get(tsd) += usize;
-		ta = thread_allocated_tsd_get();
+		*tsd_thread_deallocatedp_get(tsd) += old_usize;
-		ta->allocated += usize;
-		ta->deallocated += old_usize;
 	}
 	UTRACE(ptr, size, p);
 	JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize,
@@ -1724,8 +1745,8 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
 }
 
 JEMALLOC_ALWAYS_INLINE_C size_t
-ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
+ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
-    size_t alignment, bool zero, arena_t *arena)
+    size_t extra, size_t alignment, bool zero, arena_t *arena)
 {
 	size_t max_usize, usize;
 	prof_tctx_t *old_tctx, *tctx;
@@ -1739,7 +1760,7 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
 	 */
 	max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra,
 	    alignment);
-	tctx = prof_alloc_prep(max_usize, false);
+	tctx = prof_alloc_prep(tsd, max_usize, false);
 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
 		usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
 		    alignment, zero, max_usize, arena, tctx);
@@ -1748,10 +1769,10 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
 		    zero, arena);
 	}
 	if (unlikely(usize == old_usize)) {
-		prof_alloc_rollback(tctx, false);
+		prof_alloc_rollback(tsd, tctx, false);
 		return (usize);
 	}
-	prof_realloc(ptr, usize, tctx, false, old_usize, old_tctx);
+	prof_realloc(tsd, ptr, usize, tctx, false, old_usize, old_tctx);
 
 	return (usize);
 }
@@ -1759,6 +1780,7 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
 size_t
 je_xallocx(void *ptr, size_t size, size_t extra, int flags)
 {
+	tsd_t *tsd;
 	size_t usize, old_usize;
 	UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
 	size_t alignment = MALLOCX_ALIGN_GET(flags);
@@ -1778,12 +1800,16 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
 		arena = NULL;
 
 	old_usize = isalloc(ptr, config_prof);
+	if (unlikely((tsd = tsd_tryget()) == NULL)) {
+		usize = old_usize;
+		goto label_not_resized;
+	}
 	if (config_valgrind && unlikely(in_valgrind))
 		old_rzsize = u2rz(old_usize);
 
 	if (config_prof && opt_prof) {
-		usize = ixallocx_prof(ptr, old_usize, size, extra, alignment,
+		usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
-		    zero, arena);
+		    alignment, zero, arena);
 	} else {
 		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
 		    zero, arena);
@@ -1792,10 +1818,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
 		goto label_not_resized;
 
 	if (config_stats) {
-		thread_allocated_t *ta;
+		*tsd_thread_allocatedp_get(tsd) += usize;
-		ta = thread_allocated_tsd_get();
+		*tsd_thread_deallocatedp_get(tsd) += old_usize;
-		ta->allocated += usize;
-		ta->deallocated += old_usize;
 	}
 	JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize,
 	    old_rzsize, false, zero);
@@ -1839,7 +1863,7 @@ je_dallocx(void *ptr, int flags)
 		try_tcache = true;
 
 	UTRACE(ptr, 0, 0);
-	ifree(ptr, try_tcache);
+	ifree(tsd_tryget(), ptr, try_tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE_C size_t
@@ -1875,7 +1899,7 @@ je_sdallocx(void *ptr, size_t size, int flags)
 		try_tcache = true;
 
 	UTRACE(ptr, 0, 0);
-	isfree(ptr, usize, try_tcache);
+	isfree(tsd_tryget(), ptr, usize, try_tcache);
 }
 
 size_t
@@ -2072,9 +2096,9 @@ a0alloc(size_t size, bool zero)
 		size = 1;
 
 	if (size <= arena_maxclass)
-		return (arena_malloc(arenas[0], size, zero, false));
+		return (arena_malloc(NULL, arenas[0], size, zero, false));
 	else
-		return (huge_malloc(NULL, size, zero));
+		return (huge_malloc(NULL, arenas[0], size, zero));
 }
 
 void *
@@ -2101,7 +2125,7 @@ a0free(void *ptr)
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	if (chunk != ptr)
-		arena_dalloc(chunk, ptr, false);
+		arena_dalloc(NULL, chunk, ptr, false);
 	else
 		huge_dalloc(ptr);
 }

src/prof.c

@@ -14,8 +14,6 @@
 /******************************************************************************/
 /* Data. */
 
-malloc_tsd_data(, prof_tdata, prof_tdata_t *, NULL)
-
 bool		opt_prof = false;
 bool		opt_prof_active = true;
 size_t		opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
@@ -102,9 +100,9 @@ static bool		prof_booted = false;
  */
 
 static bool	prof_tctx_should_destroy(prof_tctx_t *tctx);
-static void	prof_tctx_destroy(prof_tctx_t *tctx);
+static void	prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
 static bool	prof_tdata_should_destroy(prof_tdata_t *tdata);
-static void	prof_tdata_destroy(prof_tdata_t *tdata);
+static void	prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata);
 
 /******************************************************************************/
 /* Red-black trees. */
@@ -151,7 +149,7 @@ rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
 /******************************************************************************/
 
 void
-prof_alloc_rollback(prof_tctx_t *tctx, bool updated)
+prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
 {
 	prof_tdata_t *tdata;
 
@@ -164,8 +162,8 @@ prof_alloc_rollback(prof_tctx_t *tctx, bool updated)
 		 * potential for sample bias is minimal except in contrived
 		 * programs.
 		 */
-		tdata = prof_tdata_get(true);
+		tdata = prof_tdata_get(tsd, true);
-		if ((uintptr_t)tdata > (uintptr_t)PROF_TDATA_STATE_MAX)
+		if (tdata != NULL)
 			prof_sample_threshold_update(tctx->tdata);
 	}
 
@@ -173,7 +171,7 @@ prof_alloc_rollback(prof_tctx_t *tctx, bool updated)
 		malloc_mutex_lock(tctx->tdata->lock);
 		tctx->prepared = false;
 		if (prof_tctx_should_destroy(tctx))
-			prof_tctx_destroy(tctx);
+			prof_tctx_destroy(tsd, tctx);
 		else
 			malloc_mutex_unlock(tctx->tdata->lock);
 	}
@@ -195,7 +193,7 @@ prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) {
 }
 
 void
-prof_free_sampled_object(size_t usize, prof_tctx_t *tctx)
+prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
 {
 
 	malloc_mutex_lock(tctx->tdata->lock);
@@ -205,7 +203,7 @@ prof_free_sampled_object(size_t usize, prof_tctx_t *tctx)
 	tctx->cnts.curbytes -= usize;
 
 	if (prof_tctx_should_destroy(tctx))
-		prof_tctx_destroy(tctx);
+		prof_tctx_destroy(tsd, tctx);
 	else
 		malloc_mutex_unlock(tctx->tdata->lock);
 }
@@ -494,13 +492,13 @@ prof_tdata_mutex_choose(uint64_t thr_uid)
 }
 
 static prof_gctx_t *
-prof_gctx_create(prof_bt_t *bt)
+prof_gctx_create(tsd_t *tsd, prof_bt_t *bt)
 {
 	/*
 	 * Create a single allocation that has space for vec of length bt->len.
 	 */
-	prof_gctx_t *gctx = (prof_gctx_t *)imalloc(offsetof(prof_gctx_t, vec) +
+	prof_gctx_t *gctx = (prof_gctx_t *)imalloc(tsd, offsetof(prof_gctx_t,
-	    (bt->len * sizeof(void *)));
+	    vec) + (bt->len * sizeof(void *)));
 	if (gctx == NULL)
 		return (NULL);
 	gctx->lock = prof_gctx_mutex_choose();
@@ -518,7 +516,7 @@ prof_gctx_create(prof_bt_t *bt)
 }
 
 static void
-prof_gctx_maybe_destroy(prof_gctx_t *gctx, prof_tdata_t *tdata)
+prof_gctx_maybe_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata)
 {
 
 	cassert(config_prof);
@@ -534,12 +532,12 @@ prof_gctx_maybe_destroy(prof_gctx_t *gctx, prof_tdata_t *tdata)
 	malloc_mutex_lock(gctx->lock);
 	if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
 		/* Remove gctx from bt2gctx. */
-		if (ckh_remove(&bt2gctx, &gctx->bt, NULL, NULL))
+		if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL))
 			not_reached();
 		prof_leave(tdata);
 		/* Destroy gctx. */
 		malloc_mutex_unlock(gctx->lock);
-		idalloc(gctx);
+		idalloc(tsd, gctx);
 	} else {
 		/*
 		 * Compensate for increment in prof_tctx_destroy() or
@@ -580,7 +578,7 @@ prof_gctx_should_destroy(prof_gctx_t *gctx)
 
 /* tctx->tdata->lock is held upon entry, and released before return. */
 static void
-prof_tctx_destroy(prof_tctx_t *tctx)
+prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
 {
 	prof_tdata_t *tdata = tctx->tdata;
 	prof_gctx_t *gctx = tctx->gctx;
@@ -592,7 +590,7 @@ prof_tctx_destroy(prof_tctx_t *tctx)
 	assert(tctx->cnts.accumobjs == 0);
 	assert(tctx->cnts.accumbytes == 0);
 
-	ckh_remove(&tdata->bt2tctx, &gctx->bt, NULL, NULL);
+	ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
 	destroy_tdata = prof_tdata_should_destroy(tdata);
 	malloc_mutex_unlock(tdata->lock);
 
@@ -618,17 +616,17 @@ prof_tctx_destroy(prof_tctx_t *tctx)
 		destroy_gctx = false;
 	malloc_mutex_unlock(gctx->lock);
 	if (destroy_gctx)
-		prof_gctx_maybe_destroy(gctx, tdata);
+		prof_gctx_maybe_destroy(tsd, gctx, tdata);
 
 	if (destroy_tdata)
-		prof_tdata_destroy(tdata);
+		prof_tdata_destroy(tsd, tdata);
 
-	idalloc(tctx);
+	idalloc(tsd, tctx);
 }
 
 static bool
-prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey,
+prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
-    prof_gctx_t **p_gctx, bool *p_new_gctx)
+    void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx)
 {
 	union {
 		prof_gctx_t	*p;
@@ -643,16 +641,16 @@ prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey,
 	prof_enter(tdata);
 	if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
 		/* bt has never been seen before.  Insert it. */
-		gctx.p = prof_gctx_create(bt);
+		gctx.p = prof_gctx_create(tsd, bt);
 		if (gctx.v == NULL) {
 			prof_leave(tdata);
 			return (true);
 		}
 		btkey.p = &gctx.p->bt;
-		if (ckh_insert(&bt2gctx, btkey.v, gctx.v)) {
+		if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
 			/* OOM. */
 			prof_leave(tdata);
-			idalloc(gctx.v);
+			idalloc(tsd, gctx.v);
 			return (true);
 		}
 		new_gctx = true;
@@ -675,7 +673,7 @@ prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey,
 }
 
 prof_tctx_t *
-prof_lookup(prof_bt_t *bt)
+prof_lookup(tsd_t *tsd, prof_bt_t *bt)
 {
 	union {
 		prof_tctx_t	*p;
@@ -686,8 +684,8 @@ prof_lookup(prof_bt_t *bt)
 
 	cassert(config_prof);
 
-	tdata = prof_tdata_get(false);
+	tdata = prof_tdata_get(tsd, false);
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+	if (tdata == NULL)
 		return (NULL);
 
 	malloc_mutex_lock(tdata->lock);
@@ -704,15 +702,15 @@ prof_lookup(prof_bt_t *bt)
 		 * This thread's cache lacks bt.  Look for it in the global
 		 * cache.
 		 */
-		if (prof_lookup_global(bt, tdata, &btkey, &gctx,
+		if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
 		    &new_gctx))
 			return (NULL);
 
 		/* Link a prof_tctx_t into gctx for this thread. */
-		ret.v = imalloc(sizeof(prof_tctx_t));
+		ret.v = imalloc(tsd, sizeof(prof_tctx_t));
 		if (ret.p == NULL) {
 			if (new_gctx)
-				prof_gctx_maybe_destroy(gctx, tdata);
+				prof_gctx_maybe_destroy(tsd, gctx, tdata);
 			return (NULL);
 		}
 		ret.p->tdata = tdata;
@@ -721,12 +719,12 @@ prof_lookup(prof_bt_t *bt)
 		ret.p->prepared = true;
 		ret.p->state = prof_tctx_state_nominal;
 		malloc_mutex_lock(tdata->lock);
-		error = ckh_insert(&tdata->bt2tctx, btkey, ret.v);
+		error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
 		malloc_mutex_unlock(tdata->lock);
 		if (error) {
 			if (new_gctx)
-				prof_gctx_maybe_destroy(gctx, tdata);
+				prof_gctx_maybe_destroy(tsd, gctx, tdata);
-			idalloc(ret.v);
+			idalloc(tsd, ret.v);
 			return (NULL);
 		}
 		malloc_mutex_lock(gctx->lock);
@@ -798,10 +796,13 @@ size_t
 prof_bt_count(void)
 {
 	size_t bt_count;
+	tsd_t *tsd;
 	prof_tdata_t *tdata;
 
-	tdata = prof_tdata_get(false);
+	if ((tsd = tsd_tryget()) == NULL)
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+		return (0);
+	tdata = prof_tdata_get(tsd, false);
+	if (tdata == NULL)
 		return (0);
 
 	prof_enter(tdata);
@@ -989,6 +990,7 @@ static prof_tctx_t *
 prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
 {
 	prof_tctx_t *ret;
+	tsd_t *tsd = (tsd_t *)arg;
 
 	switch (tctx->state) {
 	case prof_tctx_state_nominal:
@@ -1000,7 +1002,7 @@ prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
 	case prof_tctx_state_purgatory:
 		ret = tctx_tree_next(tctxs, tctx);
 		tctx_tree_remove(tctxs, tctx);
-		idalloc(tctx);
+		idalloc(tsd, tctx);
 		goto label_return;
 	default:
 		not_reached();
@@ -1049,7 +1051,8 @@ prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
 static prof_gctx_t *
 prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
 {
-	prof_tdata_t *tdata = (prof_tdata_t *)arg;
+	tsd_t *tsd = (tsd_t *)arg;
+	prof_tdata_t *tdata = prof_tdata_get(tsd, false);
 	prof_tctx_t *next;
 	bool destroy_gctx;
 
@@ -1057,13 +1060,13 @@ prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
 	next = NULL;
 	do {
 		next = tctx_tree_iter(&gctx->tctxs, next, prof_tctx_finish_iter,
-		    NULL);
+		    tsd);
 	} while (next != NULL);
 	gctx->nlimbo--;
 	destroy_gctx = prof_gctx_should_destroy(gctx);
 	malloc_mutex_unlock(gctx->lock);
 	if (destroy_gctx)
-		prof_gctx_maybe_destroy(gctx, tdata);
+		prof_gctx_maybe_destroy(tsd, gctx, tdata);
 
 	return (NULL);
 }
@@ -1277,7 +1280,7 @@ label_return:
 }
 
 static bool
-prof_dump(bool propagate_err, const char *filename, bool leakcheck)
+prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
 {
 	prof_tdata_t *tdata;
 	prof_cnt_t cnt_all;
@@ -1291,8 +1294,8 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
 
 	cassert(config_prof);
 
-	tdata = prof_tdata_get(false);
+	tdata = prof_tdata_get(tsd, false);
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+	if (tdata == NULL)
 		return (true);
 
 	malloc_mutex_lock(&prof_dump_mtx);
@@ -1341,7 +1344,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
 	if (prof_dump_close(propagate_err))
 		goto label_open_close_error;
 
-	gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tdata);
+	gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tsd);
 	malloc_mutex_unlock(&prof_dump_mtx);
 
 	if (leakcheck)
@@ -1351,7 +1354,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
 label_write_error:
 	prof_dump_close(propagate_err);
 label_open_close_error:
-	gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tdata);
+	gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tsd);
 	malloc_mutex_unlock(&prof_dump_mtx);
 	return (true);
 }
@@ -1381,24 +1384,28 @@ prof_dump_filename(char *filename, char v, uint64_t vseq)
 static void
 prof_fdump(void)
 {
+	tsd_t *tsd;
 	char filename[DUMP_FILENAME_BUFSIZE];
 
 	cassert(config_prof);
 
 	if (prof_booted == false)
 		return;
+	if ((tsd = tsd_tryget()) == NULL)
+		return;
 
 	if (opt_prof_final && opt_prof_prefix[0] != '\0') {
 		malloc_mutex_lock(&prof_dump_seq_mtx);
 		prof_dump_filename(filename, 'f', VSEQ_INVALID);
 		malloc_mutex_unlock(&prof_dump_seq_mtx);
-		prof_dump(false, filename, opt_prof_leak);
+		prof_dump(tsd, false, filename, opt_prof_leak);
 	}
 }
 
 void
 prof_idump(void)
 {
+	tsd_t *tsd;
 	prof_tdata_t *tdata;
 	char filename[PATH_MAX + 1];
 
@@ -1406,8 +1413,10 @@ prof_idump(void)
 
 	if (prof_booted == false)
 		return;
-	tdata = prof_tdata_get(false);
+	if ((tsd = tsd_tryget()) == NULL)
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+		return;
+	tdata = prof_tdata_get(tsd, false);
+	if (tdata == NULL)
 		return;
 	if (tdata->enq) {
 		tdata->enq_idump = true;
@@ -1419,19 +1428,22 @@ prof_idump(void)
 		prof_dump_filename(filename, 'i', prof_dump_iseq);
 		prof_dump_iseq++;
 		malloc_mutex_unlock(&prof_dump_seq_mtx);
-		prof_dump(false, filename, false);
+		prof_dump(tsd, false, filename, false);
 	}
 }
 
 bool
 prof_mdump(const char *filename)
 {
+	tsd_t *tsd;
 	char filename_buf[DUMP_FILENAME_BUFSIZE];
 
 	cassert(config_prof);
 
 	if (opt_prof == false || prof_booted == false)
 		return (true);
+	if ((tsd = tsd_tryget()) == NULL)
+		return (true);
 
 	if (filename == NULL) {
 		/* No filename specified, so automatically generate one. */
@@ -1443,12 +1455,13 @@ prof_mdump(const char *filename)
 		malloc_mutex_unlock(&prof_dump_seq_mtx);
 		filename = filename_buf;
 	}
-	return (prof_dump(true, filename, false));
+	return (prof_dump(tsd, true, filename, false));
 }
 
 void
 prof_gdump(void)
 {
+	tsd_t *tsd;
 	prof_tdata_t *tdata;
 	char filename[DUMP_FILENAME_BUFSIZE];
 
@@ -1456,8 +1469,10 @@ prof_gdump(void)
 
 	if (prof_booted == false)
 		return;
-	tdata = prof_tdata_get(false);
+	if ((tsd = tsd_tryget()) == NULL)
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+		return;
+	tdata = prof_tdata_get(tsd, false);
+	if (tdata == NULL)
 		return;
 	if (tdata->enq) {
 		tdata->enq_gdump = true;
@@ -1469,7 +1484,7 @@ prof_gdump(void)
 		prof_dump_filename(filename, 'u', prof_dump_useq);
 		prof_dump_useq++;
 		malloc_mutex_unlock(&prof_dump_seq_mtx);
-		prof_dump(false, filename, false);
+		prof_dump(tsd, false, filename, false);
 	}
 }
 
@@ -1510,14 +1525,14 @@ prof_thr_uid_alloc(void)
 }
 
 static prof_tdata_t *
-prof_tdata_init_impl(uint64_t thr_uid)
+prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid)
 {
 	prof_tdata_t *tdata;
 
 	cassert(config_prof);
 
 	/* Initialize an empty cache for this thread. */
-	tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t));
+	tdata = (prof_tdata_t *)imalloc(tsd, sizeof(prof_tdata_t));
 	if (tdata == NULL)
 		return (NULL);
 
@@ -1526,9 +1541,9 @@ prof_tdata_init_impl(uint64_t thr_uid)
 	tdata->thread_name = NULL;
 	tdata->state = prof_tdata_state_attached;
 
-	if (ckh_new(&tdata->bt2tctx, PROF_CKH_MINITEMS,
+	if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS,
 	    prof_bt_hash, prof_bt_keycomp)) {
-		idalloc(tdata);
+		idalloc(tsd, tdata);
 		return (NULL);
 	}
 
@@ -1542,8 +1557,6 @@ prof_tdata_init_impl(uint64_t thr_uid)
 	tdata->dumping = false;
 	tdata->active = true;
 
-	prof_tdata_tsd_set(&tdata);
-
 	malloc_mutex_lock(&tdatas_mtx);
 	tdata_tree_insert(&tdatas, tdata);
 	malloc_mutex_unlock(&tdatas_mtx);
@@ -1552,17 +1565,17 @@ prof_tdata_init_impl(uint64_t thr_uid)
 }
 
 prof_tdata_t *
-prof_tdata_init(void)
+prof_tdata_init(tsd_t *tsd)
 {
 
-	return (prof_tdata_init_impl(prof_thr_uid_alloc()));
+	return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc()));
 }
 
 prof_tdata_t *
-prof_tdata_reinit(prof_tdata_t *tdata)
+prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata)
 {
 
-	return (prof_tdata_init_impl(tdata->thr_uid));
+	return (prof_tdata_init_impl(tsd, tdata->thr_uid));
 }
 
 /* tdata->lock must be held. */
@@ -1578,7 +1591,7 @@ prof_tdata_should_destroy(prof_tdata_t *tdata)
 }
 
 static void
-prof_tdata_destroy(prof_tdata_t *tdata)
+prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata)
 {
 
 	assert(prof_tdata_should_destroy(tdata));
@@ -1588,13 +1601,14 @@ prof_tdata_destroy(prof_tdata_t *tdata)
 	malloc_mutex_unlock(&tdatas_mtx);
 
 	if (tdata->thread_name != NULL)
-		idalloc(tdata->thread_name);
+		idalloc(tsd, tdata->thread_name);
-	ckh_delete(&tdata->bt2tctx);
+	ckh_delete(tsd, &tdata->bt2tctx);
-	idalloc(tdata);
+	idalloc(tsd, tdata);
 }
 
 static void
-prof_tdata_state_transition(prof_tdata_t *tdata, prof_tdata_state_t state)
+prof_tdata_state_transition(tsd_t *tsd, prof_tdata_t *tdata,
+   prof_tdata_state_t state)
 {
 	bool destroy_tdata;
 
@@ -1606,33 +1620,34 @@ prof_tdata_state_transition(prof_tdata_t *tdata, prof_tdata_state_t state)
 		destroy_tdata = false;
 	malloc_mutex_unlock(tdata->lock);
 	if (destroy_tdata)
-		prof_tdata_destroy(tdata);
+		prof_tdata_destroy(tsd, tdata);
 }
 
 static void
-prof_tdata_detach(prof_tdata_t *tdata)
+prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
 {
 
-	prof_tdata_state_transition(tdata, prof_tdata_state_detached);
+	prof_tdata_state_transition(tsd, tdata, prof_tdata_state_detached);
 }
 
 static void
-prof_tdata_expire(prof_tdata_t *tdata)
+prof_tdata_expire(tsd_t *tsd, prof_tdata_t *tdata)
 {
 
-	prof_tdata_state_transition(tdata, prof_tdata_state_expired);
+	prof_tdata_state_transition(tsd, tdata, prof_tdata_state_expired);
 }
 
 static prof_tdata_t *
 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
 {
+	tsd_t *tsd = (tsd_t *)arg;
 
-	prof_tdata_expire(tdata);
+	prof_tdata_expire(tsd, tdata);
 	return (NULL);
 }
 
 void
-prof_reset(size_t lg_sample)
+prof_reset(tsd_t *tsd, size_t lg_sample)
 {
 
 	assert(lg_sample < (sizeof(uint64_t) << 3));
@@ -1641,69 +1656,58 @@ prof_reset(size_t lg_sample)
 	malloc_mutex_lock(&tdatas_mtx);
 
 	lg_prof_sample = lg_sample;
-	tdata_tree_iter(&tdatas, NULL, prof_tdata_reset_iter, NULL);
+	tdata_tree_iter(&tdatas, NULL, prof_tdata_reset_iter, tsd);
 
 	malloc_mutex_unlock(&tdatas_mtx);
 	malloc_mutex_unlock(&prof_dump_mtx);
 }
 
 void
-prof_tdata_cleanup(void *arg)
+prof_tdata_cleanup(tsd_t *tsd)
 {
-	prof_tdata_t *tdata = *(prof_tdata_t **)arg;
+	prof_tdata_t *tdata;
 
-	cassert(config_prof);
+	if (!config_prof)
+		return;
 
-	if (tdata == PROF_TDATA_STATE_REINCARNATED) {
+	tdata = tsd_prof_tdata_get(tsd);
-		/*
+	if (tdata != NULL)
-		 * Another destructor deallocated memory after this destructor
+		prof_tdata_detach(tsd, tdata);
-		 * was called.  Reset tdata to PROF_TDATA_STATE_PURGATORY in
-		 * order to receive another callback.
-		 */
-		tdata = PROF_TDATA_STATE_PURGATORY;
-		prof_tdata_tsd_set(&tdata);
-	} else if (tdata == PROF_TDATA_STATE_PURGATORY) {
-		/*
-		 * The previous time this destructor was called, we set the key
-		 * to PROF_TDATA_STATE_PURGATORY so that other destructors
-		 * wouldn't cause re-creation of the tdata.  This time, do
-		 * nothing, so that the destructor will not be called again.
-		 */
-	} else if (tdata != NULL) {
-		prof_tdata_detach(tdata);
-		tdata = PROF_TDATA_STATE_PURGATORY;
-		prof_tdata_tsd_set(&tdata);
-	}
 }
 
 const char *
 prof_thread_name_get(void)
 {
-	prof_tdata_t *tdata = prof_tdata_get(true);
+	tsd_t *tsd;
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+	prof_tdata_t *tdata;
+
+	if ((tsd = tsd_tryget()) == NULL)
+		return (NULL);
+	tdata = prof_tdata_get(tsd, true);
+	if (tdata == NULL)
 		return (NULL);
 	return (tdata->thread_name);
 }
 
 bool
-prof_thread_name_set(const char *thread_name)
+prof_thread_name_set(tsd_t *tsd, const char *thread_name)
 {
 	prof_tdata_t *tdata;
 	size_t size;
 	char *s;
 
-	tdata = prof_tdata_get(true);
+	tdata = prof_tdata_get(tsd, true);
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+	if (tdata == NULL)
 		return (true);
 
 	size = strlen(thread_name) + 1;
-	s = imalloc(size);
+	s = imalloc(tsd, size);
 	if (s == NULL)
 		return (true);
 
 	memcpy(s, thread_name, size);
 	if (tdata->thread_name != NULL)
-		idalloc(tdata->thread_name);
+		idalloc(tsd, tdata->thread_name);
 	tdata->thread_name = s;
 	return (false);
 }
@@ -1711,8 +1715,13 @@ prof_thread_name_set(const char *thread_name)
 bool
 prof_thread_active_get(void)
 {
-	prof_tdata_t *tdata = prof_tdata_get(true);
+	tsd_t *tsd;
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+	prof_tdata_t *tdata;
+
+	if ((tsd = tsd_tryget()) == NULL)
+		return (false);
+	tdata = prof_tdata_get(tsd, true);
+	if (tdata == NULL)
 		return (false);
 	return (tdata->active);
 }
@@ -1720,10 +1729,13 @@ prof_thread_active_get(void)
 bool
 prof_thread_active_set(bool active)
 {
+	tsd_t *tsd;
 	prof_tdata_t *tdata;
 
-	tdata = prof_tdata_get(true);
+	if ((tsd = tsd_tryget()) == NULL)
-	if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
+		return (true);
+	tdata = prof_tdata_get(tsd, true);
+	if (tdata == NULL)
 		return (true);
 	tdata->active = active;
 	return (false);
@@ -1772,20 +1784,18 @@ prof_boot2(void)
 	cassert(config_prof);
 
 	if (opt_prof) {
+		tsd_t *tsd;
 		unsigned i;
 
 		lg_prof_sample = opt_lg_prof_sample;
 
-		if (ckh_new(&bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
+		if ((tsd = tsd_tryget()) == NULL)
+			return (true);
+		if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
 		    prof_bt_keycomp))
 			return (true);
 		if (malloc_mutex_init(&bt2gctx_mtx))
 			return (true);
-		if (prof_tdata_tsd_boot()) {
-			malloc_write(
-			    "<jemalloc>: Error in pthread_key_create()\n");
-			abort();
-		}
 
 		tdata_tree_new(&tdatas);
 		if (malloc_mutex_init(&tdatas_mtx))

src/quarantine.c

@@ -9,26 +9,22 @@
 #define	QUARANTINE_STATE_PURGATORY	((quarantine_t *)(uintptr_t)2)
 #define	QUARANTINE_STATE_MAX		QUARANTINE_STATE_PURGATORY
 
-/******************************************************************************/
-/* Data. */
-
-malloc_tsd_data(, quarantine, quarantine_t *, NULL)
-
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static quarantine_t	*quarantine_grow(quarantine_t *quarantine);
+static quarantine_t	*quarantine_grow(tsd_t *tsd, quarantine_t *quarantine);
-static void	quarantine_drain_one(quarantine_t *quarantine);
+static void	quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine);
-static void	quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
+static void	quarantine_drain(tsd_t *tsd, quarantine_t *quarantine,
+    size_t upper_bound);
 
 /******************************************************************************/
 
 quarantine_t *
-quarantine_init(size_t lg_maxobjs)
+quarantine_init(tsd_t *tsd, size_t lg_maxobjs)
 {
 	quarantine_t *quarantine;
 
-	quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
+	quarantine = (quarantine_t *)imalloc(tsd, offsetof(quarantine_t, objs) +
 	    ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)));
 	if (quarantine == NULL)
 		return (NULL);
@@ -37,19 +33,17 @@ quarantine_init(size_t lg_maxobjs)
 	quarantine->first = 0;
 	quarantine->lg_maxobjs = lg_maxobjs;
 
-	quarantine_tsd_set(&quarantine);
-
 	return (quarantine);
 }
 
 static quarantine_t *
-quarantine_grow(quarantine_t *quarantine)
+quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
 {
 	quarantine_t *ret;
 
-	ret = quarantine_init(quarantine->lg_maxobjs + 1);
+	ret = quarantine_init(tsd, quarantine->lg_maxobjs + 1);
 	if (ret == NULL) {
-		quarantine_drain_one(quarantine);
+		quarantine_drain_one(tsd, quarantine);
 		return (quarantine);
 	}
 
@@ -71,17 +65,17 @@ quarantine_grow(quarantine_t *quarantine)
 		memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
 		    sizeof(quarantine_obj_t));
 	}
-	idalloc(quarantine);
+	idalloc(tsd, quarantine);
 
 	return (ret);
 }
 
 static void
-quarantine_drain_one(quarantine_t *quarantine)
+quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine)
 {
 	quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
 	assert(obj->usize == isalloc(obj->ptr, config_prof));
-	idalloc(obj->ptr);
+	idalloc(tsd, obj->ptr);
 	quarantine->curbytes -= obj->usize;
 	quarantine->curobjs--;
 	quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
@@ -89,15 +83,15 @@ quarantine_drain_one(quarantine_t *quarantine)
 }
 
 static void
-quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
+quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, size_t upper_bound)
 {
 
 	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
-		quarantine_drain_one(quarantine);
+		quarantine_drain_one(tsd, quarantine);
 }
 
 void
-quarantine(void *ptr)
+quarantine(tsd_t *tsd, void *ptr)
 {
 	quarantine_t *quarantine;
 	size_t usize = isalloc(ptr, config_prof);
@@ -105,17 +99,8 @@ quarantine(void *ptr)
 	cassert(config_fill);
 	assert(opt_quarantine);
 
-	quarantine = *quarantine_tsd_get();
+	if ((quarantine = tsd_quarantine_get(tsd)) == NULL) {
-	if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
+		idalloc(tsd, ptr);
-		if (quarantine == QUARANTINE_STATE_PURGATORY) {
-			/*
-			 * Make a note that quarantine() was called after
-			 * quarantine_cleanup() was called.
-			 */
-			quarantine = QUARANTINE_STATE_REINCARNATED;
-			quarantine_tsd_set(&quarantine);
-		}
-		idalloc(ptr);
 		return;
 	}
 	/*
@@ -125,11 +110,11 @@ quarantine(void *ptr)
 	if (quarantine->curbytes + usize > opt_quarantine) {
 		size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
 		    - usize : 0;
-		quarantine_drain(quarantine, upper_bound);
+		quarantine_drain(tsd, quarantine, upper_bound);
 	}
 	/* Grow the quarantine ring buffer if it's full. */
 	if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
-		quarantine = quarantine_grow(quarantine);
+		quarantine = quarantine_grow(tsd, quarantine);
 	/* quarantine_grow() must free a slot if it fails to grow. */
 	assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
 	/* Append ptr if its size doesn't exceed the quarantine size. */
@@ -154,46 +139,22 @@ quarantine(void *ptr)
 		}
 	} else {
 		assert(quarantine->curbytes == 0);
-		idalloc(ptr);
+		idalloc(tsd, ptr);
 	}
 }
 
 void
-quarantine_cleanup(void *arg)
+quarantine_cleanup(tsd_t *tsd)
 {
-	quarantine_t *quarantine = *(quarantine_t **)arg;
+	quarantine_t *quarantine;
 
-	if (quarantine == QUARANTINE_STATE_REINCARNATED) {
+	if (!config_fill)
-		/*
+		return;
-		 * Another destructor deallocated memory after this destructor
+
-		 * was called.  Reset quarantine to QUARANTINE_STATE_PURGATORY
+	quarantine = tsd_quarantine_get(tsd);
-		 * in order to receive another callback.
+	if (quarantine != NULL) {
-		 */
+		quarantine_drain(tsd, quarantine, 0);
-		quarantine = QUARANTINE_STATE_PURGATORY;
+		idalloc(tsd, quarantine);
-		quarantine_tsd_set(&quarantine);
+		tsd_quarantine_set(tsd, NULL);
-	} else if (quarantine == QUARANTINE_STATE_PURGATORY) {
-		/*
-		 * The previous time this destructor was called, we set the key
-		 * to QUARANTINE_STATE_PURGATORY so that other destructors
-		 * wouldn't cause re-creation of the quarantine.  This time, do
-		 * nothing, so that the destructor will not be called again.
-		 */
-	} else if (quarantine != NULL) {
-		quarantine_drain(quarantine, 0);
-		idalloc(quarantine);
-		quarantine = QUARANTINE_STATE_PURGATORY;
-		quarantine_tsd_set(&quarantine);
 	}
 }
-
-bool
-quarantine_boot(void)
-{
-
-	cassert(config_fill);
-
-	if (quarantine_tsd_boot())
-		return (true);
-
-	return (false);
-}

src/rtree.c

@@ -9,8 +9,10 @@ rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc)
 
 	assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3));
 
-	bits_per_level = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1;
+	bits_per_level = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void
-	bits_in_leaf = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1;
+	    *)))) - 1;
+	bits_in_leaf = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE /
+	    sizeof(uint8_t)))) - 1;
 	if (bits > bits_in_leaf) {
 		height = 1 + (bits - bits_in_leaf) / bits_per_level;
 		if ((height-1) * bits_per_level + bits_in_leaf != bits)

src/tcache.c

@@ -4,9 +4,6 @@
 /******************************************************************************/
 /* Data. */
 
-malloc_tsd_data(, tcache, tcache_t *, NULL)
-malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default)
-
 bool	opt_tcache = true;
 ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
 
@@ -262,43 +259,14 @@ tcache_arena_dissociate(tcache_t *tcache)
 }
 
 tcache_t *
-tcache_get_hard(tcache_t *tcache, bool create)
+tcache_get_hard(tsd_t *tsd)
 {
 
-	if (tcache == NULL) {
-		if (create == false) {
-			/*
-			 * Creating a tcache here would cause
-			 * allocation as a side effect of free().
-			 * Ordinarily that would be okay since
-			 * tcache_create() failure is a soft failure
-			 * that doesn't propagate.  However, if TLS
-			 * data are freed via free() as in glibc,
-			 * subtle corruption could result from setting
-			 * a TLS variable after its backing memory is
-			 * freed.
-			 */
-			return (NULL);
-		}
 	if (tcache_enabled_get() == false) {
 		tcache_enabled_set(false); /* Memoize. */
 		return (NULL);
 	}
-		return (tcache_create(choose_arena(NULL)));
+	return (tcache_create(choose_arena(tsd, NULL)));
-	}
-	if (tcache == TCACHE_STATE_PURGATORY) {
-		/*
-		 * Make a note that an allocator function was called
-		 * after tcache_thread_cleanup() was called.
-		 */
-		tcache = TCACHE_STATE_REINCARNATED;
-		tcache_tsd_set(&tcache);
-		return (NULL);
-	}
-	if (tcache == TCACHE_STATE_REINCARNATED)
-		return (NULL);
-	not_reached();
-	return (NULL);
 }
 
 tcache_t *
@@ -328,7 +296,7 @@ tcache_create(arena_t *arena)
 	else if (size <= tcache_maxclass)
 		tcache = (tcache_t *)arena_malloc_large(arena, size, true);
 	else
-		tcache = (tcache_t *)icalloct(size, false, arena);
+		tcache = (tcache_t *)icalloct(NULL, size, false, arena);
 
 	if (tcache == NULL)
 		return (NULL);
@@ -343,13 +311,11 @@ tcache_create(arena_t *arena)
 		stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
 	}
 
-	tcache_tsd_set(&tcache);
-
 	return (tcache);
 }
 
-void
+static void
-tcache_destroy(tcache_t *tcache)
+tcache_destroy(tsd_t *tsd, tcache_t *tcache)
 {
 	unsigned i;
 	size_t tcache_size;
@@ -403,39 +369,30 @@ tcache_destroy(tcache_t *tcache)
 
 		arena_dalloc_large(arena, chunk, tcache);
 	} else
-		idalloct(tcache, false);
+		idalloct(tsd, tcache, false);
 }
 
 void
-tcache_thread_cleanup(void *arg)
+tcache_cleanup(tsd_t *tsd)
 {
-	tcache_t *tcache = *(tcache_t **)arg;
+	tcache_t *tcache;
 
-	if (tcache == TCACHE_STATE_DISABLED) {
+	if (!config_tcache)
-		/* Do nothing. */
+		return;
-	} else if (tcache == TCACHE_STATE_REINCARNATED) {
+
-		/*
+	if ((tcache = tsd_tcache_get(tsd)) != NULL) {
-		 * Another destructor called an allocator function after this
+		tcache_destroy(tsd, tcache);
-		 * destructor was called.  Reset tcache to
+		tsd_tcache_set(tsd, NULL);
-		 * TCACHE_STATE_PURGATORY in order to receive another callback.
-		 */
-		tcache = TCACHE_STATE_PURGATORY;
-		tcache_tsd_set(&tcache);
-	} else if (tcache == TCACHE_STATE_PURGATORY) {
-		/*
-		 * The previous time this destructor was called, we set the key
-		 * to TCACHE_STATE_PURGATORY so that other destructors wouldn't
-		 * cause re-creation of the tcache.  This time, do nothing, so
-		 * that the destructor will not be called again.
-		 */
-	} else if (tcache != NULL) {
-		assert(tcache != TCACHE_STATE_PURGATORY);
-		tcache_destroy(tcache);
-		tcache = TCACHE_STATE_PURGATORY;
-		tcache_tsd_set(&tcache);
 	}
 }
 
+void
+tcache_enabled_cleanup(tsd_t *tsd)
+{
+
+	/* Do nothing. */
+}
+
 /* Caller must own arena->lock. */
 void
 tcache_stats_merge(tcache_t *tcache, arena_t *arena)
@@ -464,7 +421,7 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
 }
 
 bool
-tcache_boot0(void)
+tcache_boot(void)
 {
 	unsigned i;
 
@@ -504,13 +461,3 @@ tcache_boot0(void)
 
 	return (false);
 }
-
-bool
-tcache_boot1(void)
-{
-
-	if (tcache_tsd_boot() || tcache_enabled_tsd_boot())
-		return (true);
-
-	return (false);
-}

src/tsd.c

@@ -7,6 +7,8 @@
 static unsigned ncleanups;
 static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
 
+malloc_tsd_data(, , tsd_t, TSD_INITIALIZER)
+
 /******************************************************************************/
 
 void *
@@ -14,14 +16,15 @@ malloc_tsd_malloc(size_t size)
 {
 
 	/* Avoid choose_arena() in order to dodge bootstrapping issues. */
-	return (arena_malloc(arenas[0], size, false, false));
+	return (arena_malloc(NULL, arenas[0], CACHELINE_CEILING(size), false,
+	    false));
 }
 
 void
 malloc_tsd_dalloc(void *wrapper)
 {
 
-	idalloct(wrapper, false);
+	idalloct(NULL, wrapper, false);
 }
 
 void
@@ -67,10 +70,54 @@ malloc_tsd_cleanup_register(bool (*f)(void))
 }
 
 void
+tsd_cleanup(void *arg)
+{
+	tsd_t *tsd = (tsd_t *)arg;
+
+	if (tsd == NULL) {
+		/* OOM during re-initialization. */
+		return;
+	}
+
+	switch (tsd->state) {
+	case tsd_state_nominal:
+#define O(n, t)								\
+		n##_cleanup(tsd);
+MALLOC_TSD
+#undef O
+		tsd->state = tsd_state_purgatory;
+		tsd_set(tsd);
+		break;
+	case tsd_state_purgatory:
+		/*
+		 * The previous time this destructor was called, we set the
+		 * state to tsd_state_purgatory so that other destructors
+		 * wouldn't cause re-creation of the tsd.  This time, do
+		 * nothing, and do not request another callback.
+		 */
+		break;
+	case tsd_state_reincarnated:
+		/*
+		 * Another destructor deallocated memory after this destructor
+		 * was called.  Reset state to tsd_state_purgatory and request
+		 * another callback.
+		 */
+		tsd->state = tsd_state_purgatory;
+		tsd_set(tsd);
+		break;
+	default:
+		not_reached();
+	}
+}
+
+bool
 malloc_tsd_boot(void)
 {
 
 	ncleanups = 0;
+	if (tsd_boot())
+		return (true);
+	return (false);
 }
 
 #ifdef _WIN32

test/unit/ckh.c

@@ -2,20 +2,25 @@
 
 TEST_BEGIN(test_new_delete)
 {
+	tsd_t *tsd;
 	ckh_t ckh;
 
-	assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp),
+	tsd = tsd_tryget();
-	    "Unexpected ckh_new() error");
+	assert_ptr_not_null(tsd, "Unexpected tsd failure");
-	ckh_delete(&ckh);
 
-	assert_false(ckh_new(&ckh, 3, ckh_pointer_hash, ckh_pointer_keycomp),
+	assert_false(ckh_new(tsd, &ckh, 2, ckh_string_hash, ckh_string_keycomp),
 	    "Unexpected ckh_new() error");
-	ckh_delete(&ckh);
+	ckh_delete(tsd, &ckh);
+
+	assert_false(ckh_new(tsd, &ckh, 3, ckh_pointer_hash,
+	    ckh_pointer_keycomp), "Unexpected ckh_new() error");
+	ckh_delete(tsd, &ckh);
 }
 TEST_END
 
 TEST_BEGIN(test_count_insert_search_remove)
 {
+	tsd_t *tsd;
 	ckh_t ckh;
 	const char *strs[] = {
 	    "a string",
@@ -26,7 +31,10 @@ TEST_BEGIN(test_count_insert_search_remove)
 	const char *missing = "A string not in the hash table.";
 	size_t i;
 
-	assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp),
+	tsd = tsd_tryget();
+	assert_ptr_not_null(tsd, "Unexpected tsd failure");
+
+	assert_false(ckh_new(tsd, &ckh, 2, ckh_string_hash, ckh_string_keycomp),
 	    "Unexpected ckh_new() error");
 	assert_zu_eq(ckh_count(&ckh), 0,
 	    "ckh_count() should return %zu, but it returned %zu", ZU(0),
@@ -34,7 +42,7 @@ TEST_BEGIN(test_count_insert_search_remove)
 
 	/* Insert. */
 	for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) {
-		ckh_insert(&ckh, strs[i], strs[i]);
+		ckh_insert(tsd, &ckh, strs[i], strs[i]);
 		assert_zu_eq(ckh_count(&ckh), i+1,
 		    "ckh_count() should return %zu, but it returned %zu", i+1,
 		    ckh_count(&ckh));
@@ -79,7 +87,7 @@ TEST_BEGIN(test_count_insert_search_remove)
 		vp = (i & 2) ? &v.p : NULL;
 		k.p = NULL;
 		v.p = NULL;
-		assert_false(ckh_remove(&ckh, strs[i], kp, vp),
+		assert_false(ckh_remove(tsd, &ckh, strs[i], kp, vp),
 		    "Unexpected ckh_remove() error");
 
 		ks = (i & 1) ? strs[i] : (const char *)NULL;
@@ -95,20 +103,24 @@ TEST_BEGIN(test_count_insert_search_remove)
 		    ckh_count(&ckh));
 	}
 
-	ckh_delete(&ckh);
+	ckh_delete(tsd, &ckh);
 }
 TEST_END
 
 TEST_BEGIN(test_insert_iter_remove)
 {
 #define	NITEMS ZU(1000)
+	tsd_t *tsd;
 	ckh_t ckh;
 	void **p[NITEMS];
 	void *q, *r;
 	size_t i;
 
-	assert_false(ckh_new(&ckh, 2, ckh_pointer_hash, ckh_pointer_keycomp),
+	tsd = tsd_tryget();
-	    "Unexpected ckh_new() error");
+	assert_ptr_not_null(tsd, "Unexpected tsd failure");
+
+	assert_false(ckh_new(tsd, &ckh, 2, ckh_pointer_hash,
+	    ckh_pointer_keycomp), "Unexpected ckh_new() error");
 
 	for (i = 0; i < NITEMS; i++) {
 		p[i] = mallocx(i+1, 0);
@@ -119,7 +131,7 @@ TEST_BEGIN(test_insert_iter_remove)
 		size_t j;
 
 		for (j = i; j < NITEMS; j++) {
-			assert_false(ckh_insert(&ckh, p[j], p[j]),
+			assert_false(ckh_insert(tsd, &ckh, p[j], p[j]),
 			    "Unexpected ckh_insert() failure");
 			assert_false(ckh_search(&ckh, p[j], &q, &r),
 			    "Unexpected ckh_search() failure");
@@ -134,13 +146,13 @@ TEST_BEGIN(test_insert_iter_remove)
 		for (j = i + 1; j < NITEMS; j++) {
 			assert_false(ckh_search(&ckh, p[j], NULL, NULL),
 			    "Unexpected ckh_search() failure");
-			assert_false(ckh_remove(&ckh, p[j], &q, &r),
+			assert_false(ckh_remove(tsd, &ckh, p[j], &q, &r),
 			    "Unexpected ckh_remove() failure");
 			assert_ptr_eq(p[j], q, "Key pointer mismatch");
 			assert_ptr_eq(p[j], r, "Value pointer mismatch");
 			assert_true(ckh_search(&ckh, p[j], NULL, NULL),
 			    "Unexpected ckh_search() success");
-			assert_true(ckh_remove(&ckh, p[j], &q, &r),
+			assert_true(ckh_remove(tsd, &ckh, p[j], &q, &r),
 			    "Unexpected ckh_remove() success");
 		}
 
@@ -176,13 +188,13 @@ TEST_BEGIN(test_insert_iter_remove)
 	for (i = 0; i < NITEMS; i++) {
 		assert_false(ckh_search(&ckh, p[i], NULL, NULL),
 		    "Unexpected ckh_search() failure");
-		assert_false(ckh_remove(&ckh, p[i], &q, &r),
+		assert_false(ckh_remove(tsd, &ckh, p[i], &q, &r),
 		    "Unexpected ckh_remove() failure");
 		assert_ptr_eq(p[i], q, "Key pointer mismatch");
 		assert_ptr_eq(p[i], r, "Value pointer mismatch");
 		assert_true(ckh_search(&ckh, p[i], NULL, NULL),
 		    "Unexpected ckh_search() success");
-		assert_true(ckh_remove(&ckh, p[i], &q, &r),
+		assert_true(ckh_remove(tsd, &ckh, p[i], &q, &r),
 		    "Unexpected ckh_remove() success");
 		dallocx(p[i], 0);
 	}
@@ -190,7 +202,7 @@ TEST_BEGIN(test_insert_iter_remove)
 	assert_zu_eq(ckh_count(&ckh), 0,
 	    "ckh_count() should return %zu, but it returned %zu", ZU(0),
 	    ckh_count(&ckh));
-	ckh_delete(&ckh);
+	ckh_delete(tsd, &ckh);
 #undef NITEMS
 }
 TEST_END

test/unit/rtree.c

@@ -5,7 +5,7 @@ TEST_BEGIN(test_rtree_get_empty)
 	unsigned i;
 
 	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
-		rtree_t *rtree = rtree_new(i, imalloc, idalloc);
+		rtree_t *rtree = rtree_new(i, malloc, free);
 		assert_u_eq(rtree_get(rtree, 0), 0,
 		    "rtree_get() should return NULL for empty tree");
 		rtree_delete(rtree);
@@ -18,7 +18,7 @@ TEST_BEGIN(test_rtree_extrema)
 	unsigned i;
 
 	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
-		rtree_t *rtree = rtree_new(i, imalloc, idalloc);
+		rtree_t *rtree = rtree_new(i, malloc, free);
 
 		rtree_set(rtree, 0, 1);
 		assert_u_eq(rtree_get(rtree, 0), 1,
@@ -40,7 +40,7 @@ TEST_BEGIN(test_rtree_bits)
 	for (i = 1; i < (sizeof(uintptr_t) << 3); i++) {
 		uintptr_t keys[] = {0, 1,
 		    (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)) - 1};
-		rtree_t *rtree = rtree_new(i, imalloc, idalloc);
+		rtree_t *rtree = rtree_new(i, malloc, free);
 
 		for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) {
 			rtree_set(rtree, keys[j], 1);
@@ -73,7 +73,7 @@ TEST_BEGIN(test_rtree_random)
 
 	sfmt = init_gen_rand(SEED);
 	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
-		rtree_t *rtree = rtree_new(i, imalloc, idalloc);
+		rtree_t *rtree = rtree_new(i, malloc, free);
 		uintptr_t keys[NSET];
 		unsigned j;
 

test/unit/tsd.c

@@ -16,11 +16,11 @@ data_cleanup(void *arg)
 	data_cleanup_executed = true;
 }
 
-malloc_tsd_protos(, data, data_t)
+malloc_tsd_protos(, data_, data_t)
-malloc_tsd_externs(data, data_t)
+malloc_tsd_externs(data_, data_t)
 #define	DATA_INIT 0x12345678
-malloc_tsd_data(, data, data_t, DATA_INIT)
+malloc_tsd_data(, data_, data_t, DATA_INIT)
-malloc_tsd_funcs(, data, data_t, DATA_INIT, data_cleanup)
+malloc_tsd_funcs(, data_, data_t, DATA_INIT, data_cleanup)
 
 static void *
 thd_start(void *arg)