diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index ad4f9c24..c46feee3 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -1,18 +1,20 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES +typedef struct malloc_mutex_s malloc_mutex_t; + #ifdef JEMALLOC_OSSPIN -typedef OSSpinLock malloc_mutex_t; -#define MALLOC_MUTEX_INITIALIZER 0 +#define MALLOC_MUTEX_INITIALIZER {0} +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) +#define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER, NULL} #else -typedef pthread_mutex_t malloc_mutex_t; # if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) && \ defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP)) # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP -# define MALLOC_MUTEX_INITIALIZER PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +# define MALLOC_MUTEX_INITIALIZER {PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP} # else # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT -# define MALLOC_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +# define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER} # endif #endif @@ -20,6 +22,17 @@ typedef pthread_mutex_t malloc_mutex_t; /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +struct malloc_mutex_s { +#ifdef JEMALLOC_OSSPIN + OSSpinLock lock; +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) + pthread_mutex_t lock; + malloc_mutex_t *postponed_next; +#else + pthread_mutex_t lock; +#endif +}; + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS @@ -34,6 +47,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex); void malloc_mutex_prefork(malloc_mutex_t *mutex); void malloc_mutex_postfork_parent(malloc_mutex_t *mutex); void malloc_mutex_postfork_child(malloc_mutex_t *mutex); +bool mutex_boot(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -51,9 +65,9 @@ malloc_mutex_lock(malloc_mutex_t *mutex) if (isthreaded) { #ifdef JEMALLOC_OSSPIN - OSSpinLockLock(mutex); + OSSpinLockLock(&mutex->lock); #else - pthread_mutex_lock(mutex); + pthread_mutex_lock(&mutex->lock); #endif } } @@ -64,9 +78,9 @@ malloc_mutex_unlock(malloc_mutex_t *mutex) if (isthreaded) { #ifdef JEMALLOC_OSSPIN - OSSpinLockUnlock(mutex); + OSSpinLockUnlock(&mutex->lock); #else - pthread_mutex_unlock(mutex); + pthread_mutex_unlock(&mutex->lock); #endif } } diff --git a/src/jemalloc.c b/src/jemalloc.c index c7019220..a6d2df57 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -651,6 +651,11 @@ malloc_init_hard(void) return (true); } + if (mutex_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + if (opt_narenas == 0) { /* * For SMP systems, create more than one arena per CPU by diff --git a/src/mutex.c b/src/mutex.c index 0b20bbf3..4b8ce570 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -11,6 +11,10 @@ #ifdef JEMALLOC_LAZY_LOCK bool isthreaded = false; #endif +#ifdef JEMALLOC_MUTEX_INIT_CB +static bool postpone_init = true; +static malloc_mutex_t *postponed_mutexes = NULL; +#endif #ifdef JEMALLOC_LAZY_LOCK static void pthread_create_once(void); @@ -65,17 +69,23 @@ bool malloc_mutex_init(malloc_mutex_t *mutex) { #ifdef JEMALLOC_OSSPIN - *mutex = 0; + mutex->lock = 0; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) - if (_pthread_mutex_init_calloc_cb(mutex, base_calloc) != 0) - return (true); + if (postpone_init) { + mutex->postponed_next = postponed_mutexes; + postponed_mutexes = mutex; + } else { + if (_pthread_mutex_init_calloc_cb(&mutex->lock, base_calloc) != + 0) + return (true); + } #else pthread_mutexattr_t attr; if (pthread_mutexattr_init(&attr) != 0) return (true); pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE); - if (pthread_mutex_init(mutex, &attr) != 0) { + if (pthread_mutex_init(&mutex->lock, &attr) != 0) { pthread_mutexattr_destroy(&attr); return (true); } @@ -114,3 +124,19 @@ malloc_mutex_postfork_child(malloc_mutex_t *mutex) } #endif } + +bool +mutex_boot(void) +{ + +#ifdef JEMALLOC_MUTEX_INIT_CB + postpone_init = false; + while (postponed_mutexes != NULL) { + if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock, + base_calloc) != 0) + return (true); + postponed_mutexes = postponed_mutexes->postponed_next; + } +#endif + return (false); +}