Merge branch 'dev'

ChangeLog

@@ -6,6 +6,34 @@ found in the git revision history:
     http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
     git://canonware.com/jemalloc.git
 
+* 3.3.1 (March 6, 2013)
+
+  This version fixes bugs that are typically encountered only when utilizing
+  custom run-time options.
+
+  Bug fixes:
+  - Fix a locking order bug that could cause deadlock during fork if heap
+    profiling were enabled.
+  - Fix a chunk recycling bug that could cause the allocator to lose track of
+    whether a chunk was zeroed.   On FreeBSD, NetBSD, and OS X, it could cause
+    corruption if allocating via sbrk(2) (unlikely unless running with the
+    "dss:primary" option specified).  This was completely harmless on Linux
+    unless using mlockall(2) (and unlikely even then, unless the
+    --disable-munmap configure option or the "dss:primary" option was
+    specified).  This regression was introduced in 3.1.0 by the
+    mlockall(2)/madvise(2) interaction fix.
+  - Fix TLS-related memory corruption that could occur during thread exit if the
+    thread never allocated memory.  Only the quarantine and prof facilities were
+    susceptible.
+  - Fix two quarantine bugs:
+    + Internal reallocation of the quarantined object array leaked the old
+      array.
+    + Reallocation failure for internal reallocation of the quarantined object
+      array (very unlikely) resulted in memory corruption.
+  - Fix Valgrind integration to annotate all internally allocated memory in a
+    way that keeps Valgrind happy about internal data structure access.
+  - Fix building for s390 systems.
+
 * 3.3.0 (January 23, 2013)
 
   This version includes a few minor performance improvements in addition to the

include/jemalloc/internal/arena.h

@@ -463,9 +463,9 @@ void	arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind,
     size_t runind, size_t binind, size_t flags);
 void	arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
     size_t unzeroed);
-void	arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes);
+bool	arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes);
-void	arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes);
+bool	arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes);
-void	arena_prof_accum(arena_t *arena, uint64_t accumbytes);
+bool	arena_prof_accum(arena_t *arena, uint64_t accumbytes);
 size_t	arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
 size_t	arena_bin_index(arena_t *arena, arena_bin_t *bin);
 unsigned	arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
@@ -663,7 +663,7 @@ arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
 	*mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed;
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_INLINE bool
 arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes)
 {
 
@@ -672,33 +672,40 @@ arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes)
 
 	arena->prof_accumbytes += accumbytes;
 	if (arena->prof_accumbytes >= prof_interval) {
-		prof_idump();
 		arena->prof_accumbytes -= prof_interval;
+		return (true);
 	}
+	return (false);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_INLINE bool
 arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes)
 {
 
 	cassert(config_prof);
 
 	if (prof_interval == 0)
-		return;
+		return (false);
-	arena_prof_accum_impl(arena, accumbytes);
+	return (arena_prof_accum_impl(arena, accumbytes));
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_INLINE bool
 arena_prof_accum(arena_t *arena, uint64_t accumbytes)
 {
 
 	cassert(config_prof);
 
 	if (prof_interval == 0)
-		return;
+		return (false);
-	malloc_mutex_lock(&arena->lock);
+
-	arena_prof_accum_impl(arena, accumbytes);
+	{
-	malloc_mutex_unlock(&arena->lock);
+		bool ret;
+
+		malloc_mutex_lock(&arena->lock);
+		ret = arena_prof_accum_impl(arena, accumbytes);
+		malloc_mutex_unlock(&arena->lock);
+		return (ret);
+	}
 }
 
 JEMALLOC_ALWAYS_INLINE size_t

include/jemalloc/internal/jemalloc_internal.h.in

@@ -287,7 +287,7 @@ static const bool config_ivsalloc =
 #  ifdef __powerpc__
 #    define LG_QUANTUM		4
 #  endif
-#  ifdef __s390x__
+#  ifdef __s390__
 #    define LG_QUANTUM		4
 #  endif
 #  ifdef __SH4__
@@ -440,15 +440,18 @@ static const bool config_ivsalloc =
 } while (0)
 #else
 #define	RUNNING_ON_VALGRIND	((unsigned)0)
-#define	VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
+#define	VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \
-#define	VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)
+    do {} while (0)
-#define	VALGRIND_FREELIKE_BLOCK(addr, rzB)
+#define	VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \
-#define	VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len)
+    do {} while (0)
-#define	VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len)
+#define	VALGRIND_FREELIKE_BLOCK(addr, rzB) do {} while (0)
-#define	JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero)
+#define	VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr, _qzz_len) do {} while (0)
+#define	VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) do {} while (0)
+#define	VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) do {} while (0)
+#define	JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0)
 #define	JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize,	\
-    old_rzsize, zero)
+    old_rzsize, zero) do {} while (0)
-#define	JEMALLOC_VALGRIND_FREE(ptr, rzsize)
+#define	JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0)
 #endif
 
 #include "jemalloc/internal/util.h"

include/jemalloc/internal/private_namespace.h

@@ -306,7 +306,13 @@
 #define	prof_tdata_tsd_get_wrapper JEMALLOC_N(prof_tdata_tsd_get_wrapper)
 #define	prof_tdata_tsd_set JEMALLOC_N(prof_tdata_tsd_set)
 #define	quarantine JEMALLOC_N(quarantine)
+#define	quarantine_alloc_hook JEMALLOC_N(quarantine_alloc_hook)
 #define	quarantine_boot JEMALLOC_N(quarantine_boot)
+#define	quarantine_booted JEMALLOC_N(quarantine_booted)
+#define	quarantine_cleanup JEMALLOC_N(quarantine_cleanup)
+#define	quarantine_init JEMALLOC_N(quarantine_init)
+#define	quarantine_tls JEMALLOC_N(quarantine_tls)
+#define	quarantine_tsd JEMALLOC_N(quarantine_tsd)
 #define	quarantine_tsd_boot JEMALLOC_N(quarantine_tsd_boot)
 #define	quarantine_tsd_cleanup_wrapper JEMALLOC_N(quarantine_tsd_cleanup_wrapper)
 #define	quarantine_tsd_get JEMALLOC_N(quarantine_tsd_get)

include/jemalloc/internal/prof.h

@@ -237,7 +237,7 @@ void	prof_postfork_child(void);
 									\
 	assert(size == s2u(size));					\
 									\
-	prof_tdata = prof_tdata_get();					\
+	prof_tdata = prof_tdata_get(true);				\
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {	\
 		if (prof_tdata != NULL)					\
 			ret = (prof_thr_cnt_t *)(uintptr_t)1U;		\
@@ -286,7 +286,7 @@ void	prof_postfork_child(void);
 #ifndef JEMALLOC_ENABLE_INLINE
 malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
 
-prof_tdata_t	*prof_tdata_get(void);
+prof_tdata_t	*prof_tdata_get(bool create);
 void	prof_sample_threshold_update(prof_tdata_t *prof_tdata);
 prof_ctx_t	*prof_ctx_get(const void *ptr);
 void	prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
@@ -304,17 +304,15 @@ malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
     prof_tdata_cleanup)
 
 JEMALLOC_INLINE prof_tdata_t *
-prof_tdata_get(void)
+prof_tdata_get(bool create)
 {
 	prof_tdata_t *prof_tdata;
 
 	cassert(config_prof);
 
 	prof_tdata = *prof_tdata_tsd_get();
-	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {
+	if (create && prof_tdata == NULL)
-		if (prof_tdata == NULL)
+		prof_tdata = prof_tdata_init();
-			prof_tdata = prof_tdata_init();
-	}
 
 	return (prof_tdata);
 }
@@ -397,7 +395,7 @@ prof_sample_accum_update(size_t size)
 	/* Sampling logic is unnecessary if the interval is 1. */
 	assert(opt_lg_prof_sample != 0);
 
-	prof_tdata = *prof_tdata_tsd_get();
+	prof_tdata = prof_tdata_get(false);
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return (true);
 

include/jemalloc/internal/quarantine.h

@@ -1,6 +1,9 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
+typedef struct quarantine_obj_s quarantine_obj_t;
+typedef struct quarantine_s quarantine_t;
+
 /* Default per thread quarantine size if valgrind is enabled. */
 #define	JEMALLOC_VALGRIND_QUARANTINE_DEFAULT	(ZU(1) << 24)
 
@@ -8,17 +11,57 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
+struct quarantine_obj_s {
+	void	*ptr;
+	size_t	usize;
+};
+
+struct quarantine_s {
+	size_t			curbytes;
+	size_t			curobjs;
+	size_t			first;
+#define	LG_MAXOBJS_INIT 10
+	size_t			lg_maxobjs;
+	quarantine_obj_t	objs[1]; /* Dynamically sized ring buffer. */
+};
+
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
+quarantine_t	*quarantine_init(size_t lg_maxobjs);
 void	quarantine(void *ptr);
+void	quarantine_cleanup(void *arg);
 bool	quarantine_boot(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
+#ifndef JEMALLOC_ENABLE_INLINE
+malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *)
+
+void	quarantine_alloc_hook(void);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_))
+malloc_tsd_externs(quarantine, quarantine_t *)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL,
+    quarantine_cleanup)
+
+JEMALLOC_ALWAYS_INLINE void
+quarantine_alloc_hook(void)
+{
+	quarantine_t *quarantine;
+
+	assert(config_fill && opt_quarantine);
+
+	quarantine = *quarantine_tsd_get();
+	if (quarantine == NULL)
+		quarantine_init(LG_MAXOBJS_INIT);
+}
+#endif
+
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/
 

include/jemalloc/internal/tcache.h

@@ -320,8 +320,8 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 		}
 		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		memset(ret, 0, size);
-		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 	}
+	VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 	if (config_stats)
 		tbin->tstats.nrequests++;
@@ -371,8 +371,8 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 		} else {
 			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 			memset(ret, 0, size);
-			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		}
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 		if (config_stats)
 			tbin->tstats.nrequests++;

src/arena.c

@@ -366,8 +366,6 @@ arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)
 	    LG_PAGE)), (npages << LG_PAGE));
 	memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0,
 	    (npages << LG_PAGE));
-	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
-	    LG_PAGE)), (npages << LG_PAGE));
 }
 
 static inline void
@@ -380,8 +378,6 @@ arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
 	    LG_PAGE)), PAGE);
 	for (i = 0; i < PAGE / sizeof(size_t); i++)
 		assert(p[i] == 0);
-	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
-	    LG_PAGE)), PAGE);
 }
 
 static void
@@ -513,6 +509,8 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 			    run_ind+need_pages-1);
 		}
 	}
+	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
+	    LG_PAGE)), (need_pages << LG_PAGE));
 }
 
 static arena_chunk_t *
@@ -574,6 +572,11 @@ arena_chunk_alloc(arena_t *arena)
 			for (i = map_bias+1; i < chunk_npages-1; i++)
 				arena_mapbits_unzeroed_set(chunk, i, unzeroed);
 		} else if (config_debug) {
+			VALGRIND_MAKE_MEM_DEFINED(
+			    (void *)arena_mapp_get(chunk, map_bias+1),
+			    (void *)((uintptr_t)
+			    arena_mapp_get(chunk, chunk_npages-1)
+			    - (uintptr_t)arena_mapp_get(chunk, map_bias+1)));
 			for (i = map_bias+1; i < chunk_npages-1; i++) {
 				assert(arena_mapbits_unzeroed_get(chunk, i) ==
 				    unzeroed);
@@ -1246,8 +1249,6 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
 		    (uintptr_t)bin_info->bitmap_offset);
 
 		/* Initialize run internals. */
-		VALGRIND_MAKE_MEM_UNDEFINED(run, bin_info->reg0_offset -
-		    bin_info->redzone_size);
 		run->bin = bin;
 		run->nextind = 0;
 		run->nfree = bin_info->nregs;
@@ -1337,8 +1338,8 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind,
 
 	assert(tbin->ncached == 0);
 
-	if (config_prof)
+	if (config_prof && arena_prof_accum(arena, prof_accumbytes))
-		arena_prof_accum(arena, prof_accumbytes);
+		prof_idump();
 	bin = &arena->bins[binind];
 	malloc_mutex_lock(&bin->lock);
 	for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >>
@@ -1446,8 +1447,8 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 		bin->stats.nrequests++;
 	}
 	malloc_mutex_unlock(&bin->lock);
-	if (config_prof && isthreaded == false)
+	if (config_prof && isthreaded == false && arena_prof_accum(arena, size))
-		arena_prof_accum(arena, size);
+		prof_idump();
 
 	if (zero == false) {
 		if (config_fill) {
@@ -1464,8 +1465,8 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 		}
 		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		memset(ret, 0, size);
-		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 	}
+	VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 	return (ret);
 }
@@ -1474,6 +1475,7 @@ void *
 arena_malloc_large(arena_t *arena, size_t size, bool zero)
 {
 	void *ret;
+	UNUSED bool idump;
 
 	/* Large allocation. */
 	size = PAGE_CEILING(size);
@@ -1492,8 +1494,10 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero)
 		arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++;
 	}
 	if (config_prof)
-		arena_prof_accum_locked(arena, size);
+		idump = arena_prof_accum_locked(arena, size);
 	malloc_mutex_unlock(&arena->lock);
+	if (config_prof && idump)
+		prof_idump();
 
 	if (zero == false) {
 		if (config_fill) {

src/base.c

@@ -63,6 +63,7 @@ base_alloc(size_t size)
 	ret = base_next_addr;
 	base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
 	malloc_mutex_unlock(&base_mtx);
+	VALGRIND_MAKE_MEM_UNDEFINED(ret, csize);
 
 	return (ret);
 }
@@ -88,6 +89,7 @@ base_node_alloc(void)
 		ret = base_nodes;
 		base_nodes = *(extent_node_t **)ret;
 		malloc_mutex_unlock(&base_mtx);
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, sizeof(extent_node_t));
 	} else {
 		malloc_mutex_unlock(&base_mtx);
 		ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
@@ -100,6 +102,7 @@ void
 base_node_dealloc(extent_node_t *node)
 {
 
+	VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
 	malloc_mutex_lock(&base_mtx);
 	*(extent_node_t **)node = base_nodes;
 	base_nodes = node;

src/chunk.c

@@ -111,6 +111,7 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size,
 		}
 		node->addr = (void *)((uintptr_t)(ret) + size);
 		node->size = trailsize;
+		node->zeroed = zeroed;
 		extent_tree_szad_insert(chunks_szad, node);
 		extent_tree_ad_insert(chunks_ad, node);
 		node = NULL;
@@ -119,7 +120,6 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size,
 
 	if (node != NULL)
 		base_node_dealloc(node);
-	VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 	if (*zero) {
 		if (zeroed == false)
 			memset(ret, 0, size);
@@ -130,7 +130,6 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size,
 			VALGRIND_MAKE_MEM_DEFINED(ret, size);
 			for (i = 0; i < size / sizeof(size_t); i++)
 				assert(p[i] == 0);
-			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		}
 	}
 	return (ret);
@@ -179,27 +178,32 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
 	/* All strategies for allocation failed. */
 	ret = NULL;
 label_return:
-	if (config_ivsalloc && base == false && ret != NULL) {
+	if (ret != NULL) {
-		if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) {
+		if (config_ivsalloc && base == false) {
-			chunk_dealloc(ret, size, true);
+			if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) {
-			return (NULL);
+				chunk_dealloc(ret, size, true);
+				return (NULL);
+			}
 		}
-	}
+		if (config_stats || config_prof) {
-	if ((config_stats || config_prof) && ret != NULL) {
+			bool gdump;
-		bool gdump;
+			malloc_mutex_lock(&chunks_mtx);
-		malloc_mutex_lock(&chunks_mtx);
+			if (config_stats)
-		if (config_stats)
+				stats_chunks.nchunks += (size / chunksize);
-			stats_chunks.nchunks += (size / chunksize);
+			stats_chunks.curchunks += (size / chunksize);
-		stats_chunks.curchunks += (size / chunksize);
+			if (stats_chunks.curchunks > stats_chunks.highchunks) {
-		if (stats_chunks.curchunks > stats_chunks.highchunks) {
+				stats_chunks.highchunks =
-			stats_chunks.highchunks = stats_chunks.curchunks;
+				    stats_chunks.curchunks;
-			if (config_prof)
+				if (config_prof)
-				gdump = true;
+					gdump = true;
-		} else if (config_prof)
+			} else if (config_prof)
-			gdump = false;
+				gdump = false;
-		malloc_mutex_unlock(&chunks_mtx);
+			malloc_mutex_unlock(&chunks_mtx);
-		if (config_prof && opt_prof && opt_prof_gdump && gdump)
+			if (config_prof && opt_prof && opt_prof_gdump && gdump)
-			prof_gdump();
+				prof_gdump();
+		}
+		if (config_valgrind)
+			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 	}
 	assert(CHUNK_ADDR2BASE(ret) == ret);
 	return (ret);
@@ -213,6 +217,7 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 	extent_node_t *xnode, *node, *prev, key;
 
 	unzeroed = pages_purge(chunk, size);
+	VALGRIND_MAKE_MEM_NOACCESS(chunk, size);
 
 	/*
 	 * Allocate a node before acquiring chunks_mtx even though it might not

src/chunk_dss.c

@@ -127,7 +127,6 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero)
 				if (*zero) {
 					VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 					memset(ret, 0, size);
-					VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 				}
 				return (ret);
 			}

src/jemalloc.c

@@ -282,12 +282,30 @@ arenas_cleanup(void *arg)
 	malloc_mutex_unlock(&arenas_lock);
 }
 
+static JEMALLOC_ATTR(always_inline) void
+malloc_thread_init(void)
+{
+
+	/*
+	 * TSD initialization can't be safely done as a side effect of
+	 * deallocation, because it is possible for a thread to do nothing but
+	 * deallocate its TLS data via free(), in which case writing to TLS
+	 * would cause write-after-free memory corruption.  The quarantine
+	 * facility *only* gets used as a side effect of deallocation, so make
+	 * a best effort attempt at initializing its TSD by hooking all
+	 * allocation events.
+	 */
+	if (config_fill && opt_quarantine)
+		quarantine_alloc_hook();
+}
+
 static JEMALLOC_ATTR(always_inline) bool
 malloc_init(void)
 {
 
-	if (malloc_initialized == false)
+	if (malloc_initialized == false && malloc_init_hard())
-		return (malloc_init_hard());
+		return (true);
+	malloc_thread_init();
 
 	return (false);
 }
@@ -1095,6 +1113,7 @@ je_realloc(void *ptr, size_t size)
 	if (size == 0) {
 		if (ptr != NULL) {
 			/* realloc(ptr, 0) is equivalent to free(p). */
+			assert(malloc_initialized || IS_INITIALIZER);
 			if (config_prof) {
 				old_size = isalloc(ptr, true);
 				if (config_valgrind && opt_valgrind)
@@ -1120,6 +1139,7 @@ je_realloc(void *ptr, size_t size)
 
 	if (ptr != NULL) {
 		assert(malloc_initialized || IS_INITIALIZER);
+		malloc_thread_init();
 
 		if (config_prof) {
 			old_size = isalloc(ptr, true);
@@ -1323,6 +1343,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
 	size_t ret;
 
 	assert(malloc_initialized || IS_INITIALIZER);
+	malloc_thread_init();
 
 	if (config_ivsalloc)
 		ret = ivsalloc(ptr, config_prof);
@@ -1497,6 +1518,7 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
 	assert(size != 0);
 	assert(SIZE_T_MAX - size >= extra);
 	assert(malloc_initialized || IS_INITIALIZER);
+	malloc_thread_init();
 
 	if (arena_ind != UINT_MAX) {
 		arena_chunk_t *chunk;
@@ -1611,6 +1633,7 @@ je_sallocm(const void *ptr, size_t *rsize, int flags)
 	size_t sz;
 
 	assert(malloc_initialized || IS_INITIALIZER);
+	malloc_thread_init();
 
 	if (config_ivsalloc)
 		sz = ivsalloc(ptr, config_prof);
@@ -1730,12 +1753,12 @@ _malloc_prefork(void)
 
 	/* Acquire all mutexes in a safe order. */
 	ctl_prefork();
+	prof_prefork();
 	malloc_mutex_prefork(&arenas_lock);
 	for (i = 0; i < narenas_total; i++) {
 		if (arenas[i] != NULL)
 			arena_prefork(arenas[i]);
 	}
-	prof_prefork();
 	chunk_prefork();
 	base_prefork();
 	huge_prefork();
@@ -1761,12 +1784,12 @@ _malloc_postfork(void)
 	huge_postfork_parent();
 	base_postfork_parent();
 	chunk_postfork_parent();
-	prof_postfork_parent();
 	for (i = 0; i < narenas_total; i++) {
 		if (arenas[i] != NULL)
 			arena_postfork_parent(arenas[i]);
 	}
 	malloc_mutex_postfork_parent(&arenas_lock);
+	prof_postfork_parent();
 	ctl_postfork_parent();
 }
 
@@ -1781,12 +1804,12 @@ jemalloc_postfork_child(void)
 	huge_postfork_child();
 	base_postfork_child();
 	chunk_postfork_child();
-	prof_postfork_child();
 	for (i = 0; i < narenas_total; i++) {
 		if (arenas[i] != NULL)
 			arena_postfork_child(arenas[i]);
 	}
 	malloc_mutex_postfork_child(&arenas_lock);
+	prof_postfork_child();
 	ctl_postfork_child();
 }
 

src/prof.c

@@ -438,7 +438,7 @@ prof_lookup(prof_bt_t *bt)
 
 	cassert(config_prof);
 
-	prof_tdata = prof_tdata_get();
+	prof_tdata = prof_tdata_get(false);
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return (NULL);
 
@@ -684,7 +684,7 @@ prof_ctx_destroy(prof_ctx_t *ctx)
 	 * avoid a race between the main body of prof_ctx_merge() and entry
 	 * into this function.
 	 */
-	prof_tdata = *prof_tdata_tsd_get();
+	prof_tdata = prof_tdata_get(false);
 	assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX);
 	prof_enter(prof_tdata);
 	malloc_mutex_lock(ctx->lock);
@@ -844,7 +844,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
 
 	cassert(config_prof);
 
-	prof_tdata = prof_tdata_get();
+	prof_tdata = prof_tdata_get(false);
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return (true);
 	prof_enter(prof_tdata);
@@ -966,11 +966,7 @@ prof_idump(void)
 
 	if (prof_booted == false)
 		return;
-	/*
+	prof_tdata = prof_tdata_get(false);
-	 * Don't call prof_tdata_get() here, because it could cause recursive
-	 * allocation.
-	 */
-	prof_tdata = *prof_tdata_tsd_get();
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return;
 	if (prof_tdata->enq) {
@@ -1020,11 +1016,7 @@ prof_gdump(void)
 
 	if (prof_booted == false)
 		return;
-	/*
+	prof_tdata = prof_tdata_get(false);
-	 * Don't call prof_tdata_get() here, because it could cause recursive
-	 * allocation.
-	 */
-	prof_tdata = *prof_tdata_tsd_get();
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return;
 	if (prof_tdata->enq) {

src/quarantine.c

@@ -1,3 +1,4 @@
+#define	JEMALLOC_QUARANTINE_C_
 #include "jemalloc/internal/jemalloc_internal.h"
 
 /*
@@ -11,39 +12,18 @@
 /******************************************************************************/
 /* Data. */
 
-typedef struct quarantine_obj_s quarantine_obj_t;
+malloc_tsd_data(, quarantine, quarantine_t *, NULL)
-typedef struct quarantine_s quarantine_t;
-
-struct quarantine_obj_s {
-	void	*ptr;
-	size_t	usize;
-};
-
-struct quarantine_s {
-	size_t			curbytes;
-	size_t			curobjs;
-	size_t			first;
-#define	LG_MAXOBJS_INIT 10
-	size_t			lg_maxobjs;
-	quarantine_obj_t	objs[1]; /* Dynamically sized ring buffer. */
-};
-
-static void	quarantine_cleanup(void *arg);
-
-malloc_tsd_data(static, quarantine, quarantine_t *, NULL)
-malloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL,
-    quarantine_cleanup)
 
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static quarantine_t	*quarantine_init(size_t lg_maxobjs);
 static quarantine_t	*quarantine_grow(quarantine_t *quarantine);
+static void	quarantine_drain_one(quarantine_t *quarantine);
 static void	quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
 
 /******************************************************************************/
 
-static quarantine_t *
+quarantine_t *
 quarantine_init(size_t lg_maxobjs)
 {
 	quarantine_t *quarantine;
@@ -68,8 +48,10 @@ quarantine_grow(quarantine_t *quarantine)
 	quarantine_t *ret;
 
 	ret = quarantine_init(quarantine->lg_maxobjs + 1);
-	if (ret == NULL)
+	if (ret == NULL) {
+		quarantine_drain_one(quarantine);
 		return (quarantine);
+	}
 
 	ret->curbytes = quarantine->curbytes;
 	ret->curobjs = quarantine->curobjs;
@@ -89,23 +71,29 @@ quarantine_grow(quarantine_t *quarantine)
 		memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
 		    sizeof(quarantine_obj_t));
 	}
+	idalloc(quarantine);
 
 	return (ret);
 }
 
+static void
+quarantine_drain_one(quarantine_t *quarantine)
+{
+	quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
+	assert(obj->usize == isalloc(obj->ptr, config_prof));
+	idalloc(obj->ptr);
+	quarantine->curbytes -= obj->usize;
+	quarantine->curobjs--;
+	quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
+	    quarantine->lg_maxobjs) - 1);
+}
+
 static void
 quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
 {
 
-	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) {
+	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
-		quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
+		quarantine_drain_one(quarantine);
-		assert(obj->usize == isalloc(obj->ptr, config_prof));
-		idalloc(obj->ptr);
-		quarantine->curbytes -= obj->usize;
-		quarantine->curobjs--;
-		quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
-		    quarantine->lg_maxobjs) - 1);
-	}
 }
 
 void
@@ -119,24 +107,16 @@ quarantine(void *ptr)
 
 	quarantine = *quarantine_tsd_get();
 	if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
-		if (quarantine == NULL) {
+		if (quarantine == QUARANTINE_STATE_PURGATORY) {
-			if ((quarantine = quarantine_init(LG_MAXOBJS_INIT)) ==
+			/*
-			    NULL) {
+			 * Make a note that quarantine() was called after
-				idalloc(ptr);
+			 * quarantine_cleanup() was called.
-				return;
+			 */
-			}
+			quarantine = QUARANTINE_STATE_REINCARNATED;
-		} else {
+			quarantine_tsd_set(&quarantine);
-			if (quarantine == QUARANTINE_STATE_PURGATORY) {
-				/*
-				 * Make a note that quarantine() was called
-				 * after quarantine_cleanup() was called.
-				 */
-				quarantine = QUARANTINE_STATE_REINCARNATED;
-				quarantine_tsd_set(&quarantine);
-			}
-			idalloc(ptr);
-			return;
 		}
+		idalloc(ptr);
+		return;
 	}
 	/*
 	 * Drain one or more objects if the quarantine size limit would be
@@ -169,7 +149,7 @@ quarantine(void *ptr)
 	}
 }
 
-static void
+void
 quarantine_cleanup(void *arg)
 {
 	quarantine_t *quarantine = *(quarantine_t **)arg;

src/tcache.c

@@ -97,7 +97,8 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
 		arena_bin_t *bin = &arena->bins[binind];
 
 		if (config_prof && arena == tcache->arena) {
-			arena_prof_accum(arena, tcache->prof_accumbytes);
+			if (arena_prof_accum(arena, tcache->prof_accumbytes))
+				prof_idump();
 			tcache->prof_accumbytes = 0;
 		}
 
@@ -174,11 +175,14 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
 		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
 		    tbin->avail[0]);
 		arena_t *arena = chunk->arena;
+		UNUSED bool idump;
 
+		if (config_prof)
+			idump = false;
 		malloc_mutex_lock(&arena->lock);
 		if ((config_prof || config_stats) && arena == tcache->arena) {
 			if (config_prof) {
-				arena_prof_accum_locked(arena,
+				idump = arena_prof_accum_locked(arena,
 				    tcache->prof_accumbytes);
 				tcache->prof_accumbytes = 0;
 			}
@@ -210,6 +214,8 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
 			}
 		}
 		malloc_mutex_unlock(&arena->lock);
+		if (config_prof && idump)
+			prof_idump();
 	}
 	if (config_stats && merged_stats == false) {
 		/*
@@ -341,8 +347,9 @@ tcache_destroy(tcache_t *tcache)
 		}
 	}
 
-	if (config_prof && tcache->prof_accumbytes > 0)
+	if (config_prof && tcache->prof_accumbytes > 0 &&
-		arena_prof_accum(tcache->arena, tcache->prof_accumbytes);
+	    arena_prof_accum(tcache->arena, tcache->prof_accumbytes))
+		prof_idump();
 
 	tcache_size = arena_salloc(tcache, false);
 	if (tcache_size <= SMALL_MAXCLASS) {

test/ALLOCM_ARENA.c

@@ -41,6 +41,7 @@ je_thread_start(void *arg)
 		malloc_printf("Unexpected allocm() error\n");
 		abort();
 	}
+	dallocm(p, 0);
 
 	return (NULL);
 }

test/thread_arena.c

@@ -17,6 +17,7 @@ je_thread_start(void *arg)
 		malloc_printf("%s(): Error in malloc()\n", __func__);
 		return (void *)1;
 	}
+	free(p);
 
 	size = sizeof(arena_ind);
 	if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind,