diff --git a/include/jemalloc/internal/mutex_inlines.h b/include/jemalloc/internal/mutex_inlines.h index d4703d23..7adcff4e 100644 --- a/include/jemalloc/internal/mutex_inlines.h +++ b/include/jemalloc/internal/mutex_inlines.h @@ -52,11 +52,13 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { malloc_mutex_lock_slow(mutex); } /* We own the lock now. Update a few counters. */ - mutex_prof_data_t *data = &mutex->prof_data; - data->n_lock_ops++; - if (data->prev_owner != tsdn) { - data->prev_owner = tsdn; - data->n_owner_switches++; + if (config_stats) { + mutex_prof_data_t *data = &mutex->prof_data; + data->n_lock_ops++; + if (data->prev_owner != tsdn) { + data->prev_owner = tsdn; + data->n_owner_switches++; + } } } witness_lock(tsdn, &mutex->witness); diff --git a/include/jemalloc/internal/mutex_types.h b/include/jemalloc/internal/mutex_types.h index 257f69ca..3cc7bc2b 100644 --- a/include/jemalloc/internal/mutex_types.h +++ b/include/jemalloc/internal/mutex_types.h @@ -4,6 +4,12 @@ typedef struct mutex_prof_data_s mutex_prof_data_t; typedef struct malloc_mutex_s malloc_mutex_t; +/* + * Based on benchmark results, a fixed spin with this amount of retries works + * well for our critical sections. + */ +#define MALLOC_MUTEX_MAX_SPIN 250 + #ifdef _WIN32 # if _WIN32_WINNT >= 0x0600 # define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) @@ -45,20 +51,10 @@ typedef struct malloc_mutex_s malloc_mutex_t; {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL}}, \ WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} #else -/* TODO: get rid of adaptive mutex once we do our own spin. */ -# if (defined(JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) && \ - defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP)) -# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, \ - PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -# else # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT # define MALLOC_MUTEX_INITIALIZER \ {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER}}, \ WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -# endif #endif #endif /* JEMALLOC_INTERNAL_MUTEX_TYPES_H */ diff --git a/src/mutex.c b/src/mutex.c index 82a5fa3e..06ccd42c 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -69,14 +69,26 @@ void malloc_mutex_lock_slow(malloc_mutex_t *mutex) { mutex_prof_data_t *data = &mutex->prof_data; - {//TODO: a smart spin policy + if (ncpus == 1) { + goto label_spin_done; + } + + int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN; + do { + CPU_SPINWAIT; if (!malloc_mutex_trylock(mutex)) { data->n_spin_acquired++; return; } - } + } while (cnt++ < max_cnt); + if (!config_stats) { + /* Only spin is useful when stats is off. */ + malloc_mutex_lock_final(mutex); + return; + } nstime_t now, before; +label_spin_done: nstime_init(&now, 0); nstime_update(&now); nstime_copy(&before, &now);