diff --git a/include/jemalloc/internal/mutex_inlines.h b/include/jemalloc/internal/mutex_inlines.h index 7adcff4e..3a12a722 100644 --- a/include/jemalloc/internal/mutex_inlines.h +++ b/include/jemalloc/internal/mutex_inlines.h @@ -29,10 +29,11 @@ malloc_mutex_trylock(malloc_mutex_t *mutex) { /* Aggregate lock prof data. */ JEMALLOC_INLINE void malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { - sum->tot_wait_time += data->tot_wait_time; - if (data->max_wait_time > sum->max_wait_time) { - sum->max_wait_time = data->max_wait_time; + nstime_add(&sum->tot_wait_time, &data->tot_wait_time); + if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) { + nstime_copy(&sum->max_wait_time, &data->max_wait_time); } + sum->n_wait_times += data->n_wait_times; sum->n_spin_acquired += data->n_spin_acquired; diff --git a/include/jemalloc/internal/mutex_structs.h b/include/jemalloc/internal/mutex_structs.h index 8d6e7eb2..5dddb846 100644 --- a/include/jemalloc/internal/mutex_structs.h +++ b/include/jemalloc/internal/mutex_structs.h @@ -7,9 +7,9 @@ struct mutex_prof_data_s { * contention. We update them once we have the lock. */ /* Total time (in nano seconds) spent waiting on this mutex. */ - uint64_t tot_wait_time; + nstime_t tot_wait_time; /* Max time (in nano seconds) spent on a single lock operation. */ - uint64_t max_wait_time; + nstime_t max_wait_time; /* # of times have to wait for this mutex (after spinning). */ uint64_t n_wait_times; /* # of times acquired the mutex through local spinning. */ diff --git a/include/jemalloc/internal/mutex_types.h b/include/jemalloc/internal/mutex_types.h index 3cc7bc2b..bd261490 100644 --- a/include/jemalloc/internal/mutex_types.h +++ b/include/jemalloc/internal/mutex_types.h @@ -34,7 +34,8 @@ typedef struct malloc_mutex_s malloc_mutex_t; # define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0) #endif -#define LOCK_PROF_DATA_INITIALIZER {0, 0, 0, 0, 0, 0, 0, NULL, 0} +#define LOCK_PROF_DATA_INITIALIZER \ + {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, 0, 0, NULL, 0} #ifdef _WIN32 # define MALLOC_MUTEX_INITIALIZER diff --git a/src/ctl.c b/src/ctl.c index b6b031c3..360c6bdc 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -2485,9 +2485,9 @@ CTL_RO_CGEN(config_stats, stats_##n##_num_spin_acq, \ CTL_RO_CGEN(config_stats, stats_##n##_num_owner_switch, \ l.n_owner_switches, uint64_t) \ CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \ - l.tot_wait_time, uint64_t) \ + nstime_ns(&l.tot_wait_time), uint64_t) \ CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ - l.max_wait_time, uint64_t) \ + nstime_ns(&l.max_wait_time), uint64_t) \ CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ l.max_n_thds, uint64_t) diff --git a/src/mutex.c b/src/mutex.c index 06ccd42c..fa2770a3 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -68,6 +68,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void malloc_mutex_lock_slow(malloc_mutex_t *mutex) { mutex_prof_data_t *data = &mutex->prof_data; + UNUSED nstime_t before = NSTIME_ZERO_INITIALIZER; if (ncpus == 1) { goto label_spin_done; @@ -87,12 +88,11 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) { malloc_mutex_lock_final(mutex); return; } - nstime_t now, before; label_spin_done: - nstime_init(&now, 0); - nstime_update(&now); - nstime_copy(&before, &now); - + nstime_update(&before); + /* Copy before to after to avoid clock skews. */ + nstime_t after; + nstime_copy(&after, &before); uint32_t n_thds = atomic_add_u32(&data->n_waiting_thds, 1); /* One last try as above two calls may take quite some cycles. */ if (!malloc_mutex_trylock(mutex)) { @@ -103,16 +103,18 @@ label_spin_done: /* True slow path. */ malloc_mutex_lock_final(mutex); - atomic_sub_u32(&data->n_waiting_thds, 1); - nstime_update(&now); - /* Update more slow-path only counters. */ - nstime_subtract(&now, &before); - uint64_t wait_time = nstime_ns(&now); + atomic_sub_u32(&data->n_waiting_thds, 1); + nstime_update(&after); + + nstime_t delta; + nstime_copy(&delta, &after); + nstime_subtract(&delta, &before); + data->n_wait_times++; - data->tot_wait_time += wait_time; - if (wait_time > data->max_wait_time) { - data->max_wait_time = wait_time; + nstime_add(&data->tot_wait_time, &delta); + if (nstime_compare(&data->max_wait_time, &delta) < 0) { + nstime_copy(&data->max_wait_time, &delta); } if (n_thds > data->max_n_thds) { data->max_n_thds = n_thds;