Fix fork-related bugs.

Acquire/release arena bin locks as part of the prefork/postfork.  This
bug made deadlock in the child between fork and exec a possibility.

Split jemalloc_postfork() into jemalloc_postfork_{parent,child}() so
that the child can reinitialize mutexes rather than unlocking them.  In
practice, this bug tended not to cause problems.

include/jemalloc/internal/arena.h

@@ -376,6 +376,9 @@ void	*arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
     size_t alignment, bool zero);
 bool	arena_new(arena_t *arena, unsigned ind);
 void	arena_boot(void);
+void	arena_prefork(arena_t *arena);
+void	arena_postfork_parent(arena_t *arena);
+void	arena_postfork_child(arena_t *arena);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/base.h

@@ -9,12 +9,13 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-extern malloc_mutex_t	base_mtx;
-
 void	*base_alloc(size_t size);
 extent_node_t *base_node_alloc(void);
 void	base_node_dealloc(extent_node_t *node);
 bool	base_boot(void);
+void	base_prefork(void);
+void	base_postfork_parent(void);
+void	base_postfork_child(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/chunk_dss.h

@@ -9,16 +9,13 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-/*
- * Protects sbrk() calls.  This avoids malloc races among threads, though it
- * does not protect against races with threads that call sbrk() directly.
- */
-extern malloc_mutex_t	dss_mtx;
-
 void	*chunk_alloc_dss(size_t size, bool *zero);
 bool	chunk_in_dss(void *chunk);
 bool	chunk_dealloc_dss(void *chunk, size_t size);
 bool	chunk_dss_boot(void);
+void	chunk_dss_prefork(void);
+void	chunk_dss_postfork_parent(void);
+void	chunk_dss_postfork_child(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/huge.h

@@ -28,6 +28,9 @@ size_t	huge_salloc(const void *ptr);
 prof_ctx_t	*huge_prof_ctx_get(const void *ptr);
 void	huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
 bool	huge_boot(void);
+void	huge_prefork(void);
+void	huge_postfork_parent(void);
+void	huge_postfork_child(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/jemalloc_internal.h.in

@@ -410,7 +410,8 @@ thread_allocated_t	*thread_allocated_get_hard(void);
 arena_t	*arenas_extend(unsigned ind);
 arena_t	*choose_arena_hard(void);
 void	jemalloc_prefork(void);
-void	jemalloc_postfork(void);
+void	jemalloc_postfork_parent(void);
+void	jemalloc_postfork_child(void);
 
 #include "jemalloc/internal/util.h"
 #include "jemalloc/internal/atomic.h"

include/jemalloc/internal/mutex.h

@@ -29,6 +29,9 @@ extern bool isthreaded;
 
 bool	malloc_mutex_init(malloc_mutex_t *mutex);
 void	malloc_mutex_destroy(malloc_mutex_t *mutex);
+void	malloc_mutex_prefork(malloc_mutex_t *mutex);
+void	malloc_mutex_postfork_parent(malloc_mutex_t *mutex);
+void	malloc_mutex_postfork_child(malloc_mutex_t *mutex);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/private_namespace.h

@@ -8,6 +8,9 @@
 #define	arena_malloc_small JEMALLOC_N(arena_malloc_small)
 #define	arena_new JEMALLOC_N(arena_new)
 #define	arena_palloc JEMALLOC_N(arena_palloc)
+#define	arena_postfork_child JEMALLOC_N(arena_postfork_child)
+#define	arena_postfork_parent JEMALLOC_N(arena_postfork_parent)
+#define	arena_prefork JEMALLOC_N(arena_prefork)
 #define	arena_prof_accum JEMALLOC_N(arena_prof_accum)
 #define	arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get)
 #define	arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set)
@@ -32,6 +35,9 @@
 #define	base_boot JEMALLOC_N(base_boot)
 #define	base_node_alloc JEMALLOC_N(base_node_alloc)
 #define	base_node_dealloc JEMALLOC_N(base_node_dealloc)
+#define	base_postfork_child JEMALLOC_N(base_postfork_child)
+#define	base_postfork_parent JEMALLOC_N(base_postfork_parent)
+#define	base_prefork JEMALLOC_N(base_prefork)
 #define	bitmap_full JEMALLOC_N(bitmap_full)
 #define	bitmap_get JEMALLOC_N(bitmap_get)
 #define	bitmap_info_init JEMALLOC_N(bitmap_info_init)
@@ -54,6 +60,9 @@
 #define	chunk_dealloc_dss JEMALLOC_N(chunk_dealloc_dss)
 #define	chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap)
 #define	chunk_dss_boot JEMALLOC_N(chunk_dss_boot)
+#define	chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child)
+#define	chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent)
+#define	chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork)
 #define	chunk_in_dss JEMALLOC_N(chunk_in_dss)
 #define	chunk_mmap_boot JEMALLOC_N(chunk_mmap_boot)
 #define	ckh_bucket_search JEMALLOC_N(ckh_bucket_search)
@@ -115,6 +124,9 @@
 #define	huge_dalloc JEMALLOC_N(huge_dalloc)
 #define	huge_malloc JEMALLOC_N(huge_malloc)
 #define	huge_palloc JEMALLOC_N(huge_palloc)
+#define	huge_postfork_child JEMALLOC_N(huge_postfork_child)
+#define	huge_postfork_parent JEMALLOC_N(huge_postfork_parent)
+#define	huge_prefork JEMALLOC_N(huge_prefork)
 #define	huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get)
 #define	huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set)
 #define	huge_ralloc JEMALLOC_N(huge_ralloc)
@@ -129,12 +141,16 @@
 #define	isalloc JEMALLOC_N(isalloc)
 #define	ivsalloc JEMALLOC_N(ivsalloc)
 #define	jemalloc_darwin_init JEMALLOC_N(jemalloc_darwin_init)
-#define	jemalloc_postfork JEMALLOC_N(jemalloc_postfork)
+#define	jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child)
+#define	jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent)
 #define	jemalloc_prefork JEMALLOC_N(jemalloc_prefork)
 #define	malloc_cprintf JEMALLOC_N(malloc_cprintf)
 #define	malloc_mutex_destroy JEMALLOC_N(malloc_mutex_destroy)
 #define	malloc_mutex_init JEMALLOC_N(malloc_mutex_init)
 #define	malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock)
+#define	malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child)
+#define	malloc_mutex_postfork_parent JEMALLOC_N(malloc_mutex_postfork_parent)
+#define	malloc_mutex_prefork JEMALLOC_N(malloc_mutex_prefork)
 #define	malloc_mutex_trylock JEMALLOC_N(malloc_mutex_trylock)
 #define	malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock)
 #define	malloc_printf JEMALLOC_N(malloc_printf)

src/arena.c

@@ -2169,3 +2169,33 @@ arena_boot(void)
 
 	bin_info_init();
 }
+
+void
+arena_prefork(arena_t *arena)
+{
+	unsigned i;
+
+	malloc_mutex_prefork(&arena->lock);
+	for (i = 0; i < NBINS; i++)
+		malloc_mutex_prefork(&arena->bins[i].lock);
+}
+
+void
+arena_postfork_parent(arena_t *arena)
+{
+	unsigned i;
+
+	for (i = 0; i < NBINS; i++)
+		malloc_mutex_postfork_parent(&arena->bins[i].lock);
+	malloc_mutex_postfork_parent(&arena->lock);
+}
+
+void
+arena_postfork_child(arena_t *arena)
+{
+	unsigned i;
+
+	for (i = 0; i < NBINS; i++)
+		malloc_mutex_postfork_child(&arena->bins[i].lock);
+	malloc_mutex_postfork_child(&arena->lock);
+}

src/base.c

@@ -4,7 +4,7 @@
 /******************************************************************************/
 /* Data. */
 
-malloc_mutex_t	base_mtx;
+static malloc_mutex_t	base_mtx;
 
 /*
  * Current pages that are being used for internal memory allocations.  These
@@ -104,3 +104,24 @@ base_boot(void)
 
 	return (false);
 }
+
+void
+base_prefork(void)
+{
+
+	malloc_mutex_prefork(&base_mtx);
+}
+
+void
+base_postfork_parent(void)
+{
+
+	malloc_mutex_postfork_parent(&base_mtx);
+}
+
+void
+base_postfork_child(void)
+{
+
+	malloc_mutex_postfork_child(&base_mtx);
+}

src/chunk_dss.c

@@ -3,7 +3,11 @@
 /******************************************************************************/
 /* Data. */
 
-malloc_mutex_t	dss_mtx;
+/*
+ * Protects sbrk() calls.  This avoids malloc races among threads, though it
+ * does not protect against races with threads that call sbrk() directly.
+ */
+static malloc_mutex_t	dss_mtx;
 
 /* Base address of the DSS. */
 static void		*dss_base;
@@ -291,4 +295,28 @@ chunk_dss_boot(void)
 	return (false);
 }
 
+void
+chunk_dss_prefork(void)
+{
+
+	if (config_dss)
+		malloc_mutex_prefork(&dss_mtx);
+}
+
+void
+chunk_dss_postfork_parent(void)
+{
+
+	if (config_dss)
+		malloc_mutex_postfork_parent(&dss_mtx);
+}
+
+void
+chunk_dss_postfork_child(void)
+{
+
+	if (config_dss)
+		malloc_mutex_postfork_child(&dss_mtx);
+}
+
 /******************************************************************************/

src/huge.c

@@ -359,3 +359,24 @@ huge_boot(void)
 
 	return (false);
 }
+
+void
+huge_prefork(void)
+{
+
+	malloc_mutex_prefork(&huge_mtx);
+}
+
+void
+huge_postfork_parent(void)
+{
+
+	malloc_mutex_postfork_parent(&huge_mtx);
+}
+
+void
+huge_postfork_child(void)
+{
+
+	malloc_mutex_postfork_child(&huge_mtx);
+}

src/jemalloc.c

@@ -610,8 +610,8 @@ malloc_init_hard(void)
 	malloc_conf_init();
 
 	/* Register fork handlers. */
-	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork,
+	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
-	    jemalloc_postfork) != 0) {
+	    jemalloc_postfork_child) != 0) {
 		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
 		if (opt_abort)
 			abort();
@@ -1593,40 +1593,46 @@ jemalloc_prefork(void)
 	unsigned i;
 
 	/* Acquire all mutexes in a safe order. */
-
+	malloc_mutex_prefork(&arenas_lock);
-	malloc_mutex_lock(&arenas_lock);
 	for (i = 0; i < narenas; i++) {
 		if (arenas[i] != NULL)
-			malloc_mutex_lock(&arenas[i]->lock);
+			arena_prefork(arenas[i]);
 	}
-
+	base_prefork();
-	malloc_mutex_lock(&base_mtx);
+	huge_prefork();
-
+	chunk_dss_prefork();
-	malloc_mutex_lock(&huge_mtx);
-
-	if (config_dss)
-		malloc_mutex_lock(&dss_mtx);
 }
 
 void
-jemalloc_postfork(void)
+jemalloc_postfork_parent(void)
 {
 	unsigned i;
 
 	/* Release all mutexes, now that fork() has completed. */
-
+	chunk_dss_postfork_parent();
-	if (config_dss)
+	huge_postfork_parent();
-		malloc_mutex_unlock(&dss_mtx);
+	base_postfork_parent();
-
-	malloc_mutex_unlock(&huge_mtx);
-
-	malloc_mutex_unlock(&base_mtx);
-
 	for (i = 0; i < narenas; i++) {
 		if (arenas[i] != NULL)
-			malloc_mutex_unlock(&arenas[i]->lock);
+			arena_postfork_parent(arenas[i]);
 	}
-	malloc_mutex_unlock(&arenas_lock);
+	malloc_mutex_postfork_parent(&arenas_lock);
+}
+
+void
+jemalloc_postfork_child(void)
+{
+	unsigned i;
+
+	/* Release all mutexes, now that fork() has completed. */
+	chunk_dss_postfork_child();
+	huge_postfork_child();
+	base_postfork_child();
+	for (i = 0; i < narenas; i++) {
+		if (arenas[i] != NULL)
+			arena_postfork_child(arenas[i]);
+	}
+	malloc_mutex_postfork_child(&arenas_lock);
 }
 
 /******************************************************************************/

src/mutex.c

@@ -92,3 +92,29 @@ malloc_mutex_destroy(malloc_mutex_t *mutex)
 	}
 #endif
 }
+
+void
+malloc_mutex_prefork(malloc_mutex_t *mutex)
+{
+
+	malloc_mutex_lock(mutex);
+}
+
+void
+malloc_mutex_postfork_parent(malloc_mutex_t *mutex)
+{
+
+	malloc_mutex_unlock(mutex);
+}
+
+void
+malloc_mutex_postfork_child(malloc_mutex_t *mutex)
+{
+
+	if (malloc_mutex_init(mutex)) {
+		malloc_printf("<jemalloc>: Error re-initializing mutex in "
+		    "child\n");
+		if (opt_abort)
+			abort();
+	}
+}