diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 8fb1fe6d..c3c3e9cd 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -45,6 +45,9 @@ extern size_t arena_maxclass; /* Max size class for arenas. */ void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero); void chunk_dealloc(void *chunk, size_t size, bool unmap); bool chunk_boot(void); +void chunk_prefork(void); +void chunk_postfork_parent(void); +void chunk_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index adf3827f..1d0c76a0 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -75,6 +75,9 @@ int ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp); int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); bool ctl_boot(void); +void ctl_prefork(void); +void ctl_postfork_parent(void); +void ctl_postfork_child(void); #define xmallctl(name, oldp, oldlenp, newp, newlen) do { \ if (je_mallctl(name, oldp, oldlenp, newp, newlen) \ diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h index b8166470..28686dce 100644 --- a/include/jemalloc/internal/private_namespace.h +++ b/include/jemalloc/internal/private_namespace.h @@ -59,6 +59,7 @@ #define arenas_lock JEMALLOC_N(arenas_lock) #define arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index) #define arenas_tls JEMALLOC_N(arenas_tls) +#define arenas_tsd JEMALLOC_N(arenas_tsd) #define arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot) #define arenas_tsd_cleanup_wrapper JEMALLOC_N(arenas_tsd_cleanup_wrapper) #define arenas_tsd_get JEMALLOC_N(arenas_tsd_get) @@ -104,6 +105,9 @@ #define chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork) #define chunk_in_dss JEMALLOC_N(chunk_in_dss) #define chunk_npages JEMALLOC_N(chunk_npages) +#define chunk_postfork_child JEMALLOC_N(chunk_postfork_child) +#define chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent) +#define chunk_prefork JEMALLOC_N(chunk_prefork) #define chunks_mtx JEMALLOC_N(chunks_mtx) #define chunks_rtree JEMALLOC_N(chunks_rtree) #define chunksize JEMALLOC_N(chunksize) @@ -129,6 +133,9 @@ #define ctl_bymib JEMALLOC_N(ctl_bymib) #define ctl_byname JEMALLOC_N(ctl_byname) #define ctl_nametomib JEMALLOC_N(ctl_nametomib) +#define ctl_postfork_child JEMALLOC_N(ctl_postfork_child) +#define ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent) +#define ctl_prefork JEMALLOC_N(ctl_prefork) #define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first) #define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert) #define extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter) @@ -161,6 +168,7 @@ #define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse) #define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) #define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) +#define get_errno JEMALLOC_N(get_errno) #define hash JEMALLOC_N(hash) #define huge_allocated JEMALLOC_N(huge_allocated) #define huge_boot JEMALLOC_N(huge_boot) @@ -254,6 +262,9 @@ #define prof_lookup JEMALLOC_N(prof_lookup) #define prof_malloc JEMALLOC_N(prof_malloc) #define prof_mdump JEMALLOC_N(prof_mdump) +#define prof_postfork_child JEMALLOC_N(prof_postfork_child) +#define prof_postfork_parent JEMALLOC_N(prof_postfork_parent) +#define prof_prefork JEMALLOC_N(prof_prefork) #define prof_promote JEMALLOC_N(prof_promote) #define prof_realloc JEMALLOC_N(prof_realloc) #define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) @@ -264,6 +275,7 @@ #define prof_tdata_init JEMALLOC_N(prof_tdata_init) #define prof_tdata_initialized JEMALLOC_N(prof_tdata_initialized) #define prof_tdata_tls JEMALLOC_N(prof_tdata_tls) +#define prof_tdata_tsd JEMALLOC_N(prof_tdata_tsd) #define prof_tdata_tsd_boot JEMALLOC_N(prof_tdata_tsd_boot) #define prof_tdata_tsd_cleanup_wrapper JEMALLOC_N(prof_tdata_tsd_cleanup_wrapper) #define prof_tdata_tsd_get JEMALLOC_N(prof_tdata_tsd_get) @@ -278,9 +290,13 @@ #define rtree_get JEMALLOC_N(rtree_get) #define rtree_get_locked JEMALLOC_N(rtree_get_locked) #define rtree_new JEMALLOC_N(rtree_new) +#define rtree_postfork_child JEMALLOC_N(rtree_postfork_child) +#define rtree_postfork_parent JEMALLOC_N(rtree_postfork_parent) +#define rtree_prefork JEMALLOC_N(rtree_prefork) #define rtree_set JEMALLOC_N(rtree_set) #define s2u JEMALLOC_N(s2u) #define sa2u JEMALLOC_N(sa2u) +#define set_errno JEMALLOC_N(set_errno) #define stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index) #define stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index) #define stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index) @@ -311,6 +327,7 @@ #define tcache_enabled_initialized JEMALLOC_N(tcache_enabled_initialized) #define tcache_enabled_set JEMALLOC_N(tcache_enabled_set) #define tcache_enabled_tls JEMALLOC_N(tcache_enabled_tls) +#define tcache_enabled_tsd JEMALLOC_N(tcache_enabled_tsd) #define tcache_enabled_tsd_boot JEMALLOC_N(tcache_enabled_tsd_boot) #define tcache_enabled_tsd_cleanup_wrapper JEMALLOC_N(tcache_enabled_tsd_cleanup_wrapper) #define tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get) @@ -325,6 +342,7 @@ #define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) #define tcache_thread_cleanup JEMALLOC_N(tcache_thread_cleanup) #define tcache_tls JEMALLOC_N(tcache_tls) +#define tcache_tsd JEMALLOC_N(tcache_tsd) #define tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot) #define tcache_tsd_cleanup_wrapper JEMALLOC_N(tcache_tsd_cleanup_wrapper) #define tcache_tsd_get JEMALLOC_N(tcache_tsd_get) @@ -332,6 +350,7 @@ #define thread_allocated_booted JEMALLOC_N(thread_allocated_booted) #define thread_allocated_initialized JEMALLOC_N(thread_allocated_initialized) #define thread_allocated_tls JEMALLOC_N(thread_allocated_tls) +#define thread_allocated_tsd JEMALLOC_N(thread_allocated_tsd) #define thread_allocated_tsd_boot JEMALLOC_N(thread_allocated_tsd_boot) #define thread_allocated_tsd_cleanup_wrapper JEMALLOC_N(thread_allocated_tsd_cleanup_wrapper) #define thread_allocated_tsd_get JEMALLOC_N(thread_allocated_tsd_get) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 6bed90b9..47f22ad2 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -223,6 +223,9 @@ void prof_tdata_cleanup(void *arg); void prof_boot0(void); void prof_boot1(void); bool prof_boot2(void); +void prof_prefork(void); +void prof_postfork_parent(void); +void prof_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/rtree.h b/include/jemalloc/internal/rtree.h index 95d6355a..9bd98548 100644 --- a/include/jemalloc/internal/rtree.h +++ b/include/jemalloc/internal/rtree.h @@ -36,6 +36,9 @@ struct rtree_s { #ifdef JEMALLOC_H_EXTERNS rtree_t *rtree_new(unsigned bits); +void rtree_prefork(rtree_t *rtree); +void rtree_postfork_parent(rtree_t *rtree); +void rtree_postfork_child(rtree_t *rtree); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/src/chunk.c b/src/chunk.c index b43f9507..1730452f 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -318,3 +318,33 @@ chunk_boot(void) return (false); } + +void +chunk_prefork(void) +{ + + malloc_mutex_lock(&chunks_mtx); + if (config_ivsalloc) + rtree_prefork(chunks_rtree); + chunk_dss_prefork(); +} + +void +chunk_postfork_parent(void) +{ + + chunk_dss_postfork_parent(); + if (config_ivsalloc) + rtree_postfork_parent(chunks_rtree); + malloc_mutex_postfork_parent(&chunks_mtx); +} + +void +chunk_postfork_child(void) +{ + + chunk_dss_postfork_child(); + if (config_ivsalloc) + rtree_postfork_child(chunks_rtree); + malloc_mutex_postfork_child(&chunks_mtx); +} diff --git a/src/ctl.c b/src/ctl.c index 5be066a2..dec98832 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -827,6 +827,27 @@ ctl_boot(void) return (false); } +void +ctl_prefork(void) +{ + + malloc_mutex_lock(&ctl_mtx); +} + +void +ctl_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&ctl_mtx); +} + +void +ctl_postfork_child(void) +{ + + malloc_mutex_postfork_child(&ctl_mtx); +} + /******************************************************************************/ /* *_ctl() functions. */ diff --git a/src/jemalloc.c b/src/jemalloc.c index 7fa07449..4ea1f759 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1614,6 +1614,27 @@ je_nallocm(size_t *rsize, size_t size, int flags) * malloc during fork(). */ +/* + * If an application creates a thread before doing any allocation in the main + * thread, then calls fork(2) in the main thread followed by memory allocation + * in the child process, a race can occur that results in deadlock within the + * child: the main thread may have forked while the created thread had + * partially initialized the allocator. Ordinarily jemalloc prevents + * fork/malloc races via the following functions it registers during + * initialization using pthread_atfork(), but of course that does no good if + * the allocator isn't fully initialized at fork time. The following library + * constructor is a partial solution to this problem. It may still possible to + * trigger the deadlock described above, but doing so would involve forking via + * a library constructor that runs before jemalloc's runs. + */ +JEMALLOC_ATTR(constructor) +static void +jemalloc_constructor(void) +{ + + malloc_init(); +} + #ifndef JEMALLOC_MUTEX_INIT_CB void jemalloc_prefork(void) @@ -1631,14 +1652,16 @@ _malloc_prefork(void) assert(malloc_initialized); /* Acquire all mutexes in a safe order. */ + ctl_prefork(); malloc_mutex_prefork(&arenas_lock); for (i = 0; i < narenas; i++) { if (arenas[i] != NULL) arena_prefork(arenas[i]); } + prof_prefork(); base_prefork(); huge_prefork(); - chunk_dss_prefork(); + chunk_prefork(); } #ifndef JEMALLOC_MUTEX_INIT_CB @@ -1658,14 +1681,16 @@ _malloc_postfork(void) assert(malloc_initialized); /* Release all mutexes, now that fork() has completed. */ - chunk_dss_postfork_parent(); + chunk_postfork_parent(); huge_postfork_parent(); base_postfork_parent(); + prof_postfork_parent(); for (i = 0; i < narenas; i++) { if (arenas[i] != NULL) arena_postfork_parent(arenas[i]); } malloc_mutex_postfork_parent(&arenas_lock); + ctl_postfork_parent(); } void @@ -1676,14 +1701,16 @@ jemalloc_postfork_child(void) assert(malloc_initialized); /* Release all mutexes, now that fork() has completed. */ - chunk_dss_postfork_child(); + chunk_postfork_child(); huge_postfork_child(); base_postfork_child(); + prof_postfork_child(); for (i = 0; i < narenas; i++) { if (arenas[i] != NULL) arena_postfork_child(arenas[i]); } malloc_mutex_postfork_child(&arenas_lock); + ctl_postfork_child(); } /******************************************************************************/ diff --git a/src/prof.c b/src/prof.c index de1d3929..04964ef7 100644 --- a/src/prof.c +++ b/src/prof.c @@ -1270,4 +1270,46 @@ prof_boot2(void) return (false); } +void +prof_prefork(void) +{ + + if (opt_prof) { + unsigned i; + + malloc_mutex_lock(&bt2ctx_mtx); + malloc_mutex_lock(&prof_dump_seq_mtx); + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_lock(&ctx_locks[i]); + } +} + +void +prof_postfork_parent(void) +{ + + if (opt_prof) { + unsigned i; + + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_postfork_parent(&ctx_locks[i]); + malloc_mutex_postfork_parent(&prof_dump_seq_mtx); + malloc_mutex_postfork_parent(&bt2ctx_mtx); + } +} + +void +prof_postfork_child(void) +{ + + if (opt_prof) { + unsigned i; + + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_postfork_child(&ctx_locks[i]); + malloc_mutex_postfork_child(&prof_dump_seq_mtx); + malloc_mutex_postfork_child(&bt2ctx_mtx); + } +} + /******************************************************************************/ diff --git a/src/rtree.c b/src/rtree.c index eb0ff1e2..90c6935a 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -44,3 +44,24 @@ rtree_new(unsigned bits) return (ret); } + +void +rtree_prefork(rtree_t *rtree) +{ + + malloc_mutex_prefork(&rtree->mutex); +} + +void +rtree_postfork_parent(rtree_t *rtree) +{ + + malloc_mutex_postfork_parent(&rtree->mutex); +} + +void +rtree_postfork_child(rtree_t *rtree) +{ + + malloc_mutex_postfork_child(&rtree->mutex); +}