diff --git a/include/jemalloc/internal/ctl_externs.h b/include/jemalloc/internal/ctl_externs.h index 33ca2039..1b06dd4d 100644 --- a/include/jemalloc/internal/ctl_externs.h +++ b/include/jemalloc/internal/ctl_externs.h @@ -4,13 +4,13 @@ /* Maximum ctl tree depth. */ #define CTL_MAX_DEPTH 7 -#define NUM_GLOBAL_PROF_LOCKS 3 -#define NUM_ARENA_PROF_LOCKS 6 -#define NUM_LOCK_PROF_COUNTERS 7 +#define NUM_GLOBAL_PROF_MUTEXES 3 +#define NUM_ARENA_PROF_MUTEXES 6 +#define NUM_MUTEX_PROF_COUNTERS 7 -extern const char *arena_lock_names[NUM_ARENA_PROF_LOCKS]; -extern const char *global_lock_names[NUM_GLOBAL_PROF_LOCKS]; -extern const char *lock_counter_names[NUM_LOCK_PROF_COUNTERS]; +extern const char *arena_mutex_names[NUM_ARENA_PROF_MUTEXES]; +extern const char *global_mutex_names[NUM_GLOBAL_PROF_MUTEXES]; +extern const char *mutex_counter_names[NUM_MUTEX_PROF_COUNTERS]; int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen); diff --git a/include/jemalloc/internal/ctl_structs.h b/include/jemalloc/internal/ctl_structs.h index e467a247..b1ee3555 100644 --- a/include/jemalloc/internal/ctl_structs.h +++ b/include/jemalloc/internal/ctl_structs.h @@ -42,8 +42,8 @@ struct ctl_stats_s { size_t mapped; size_t retained; -#define MTX(mutex) lock_prof_data_t mutex##_mtx_data; -GLOBAL_PROF_MUTEXES +#define MTX(mutex) mutex_prof_data_t mutex##_mtx_data; + GLOBAL_PROF_MUTEXES #undef MTX }; diff --git a/include/jemalloc/internal/mutex_externs.h b/include/jemalloc/internal/mutex_externs.h index ba6418ef..5199d3cf 100644 --- a/include/jemalloc/internal/mutex_externs.h +++ b/include/jemalloc/internal/mutex_externs.h @@ -14,5 +14,6 @@ void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); bool malloc_mutex_boot(void); +void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); #endif /* JEMALLOC_INTERNAL_MUTEX_EXTERNS_H */ diff --git a/include/jemalloc/internal/mutex_inlines.h b/include/jemalloc/internal/mutex_inlines.h index 1020eefd..d4703d23 100644 --- a/include/jemalloc/internal/mutex_inlines.h +++ b/include/jemalloc/internal/mutex_inlines.h @@ -9,9 +9,9 @@ bool malloc_mutex_trylock(malloc_mutex_t *mutex); void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex); -void malloc_lock_prof_read(tsdn_t *tsdn, lock_prof_data_t *data, +void malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex); -void malloc_lock_prof_merge(lock_prof_data_t *sum, lock_prof_data_t *data); +void malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) @@ -28,7 +28,7 @@ malloc_mutex_trylock(malloc_mutex_t *mutex) { /* Aggregate lock prof data. */ JEMALLOC_INLINE void -malloc_lock_prof_merge(lock_prof_data_t *sum, lock_prof_data_t *data) { +malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { sum->tot_wait_time += data->tot_wait_time; if (data->max_wait_time > sum->max_wait_time) { sum->max_wait_time = data->max_wait_time; @@ -52,7 +52,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { malloc_mutex_lock_slow(mutex); } /* We own the lock now. Update a few counters. */ - lock_prof_data_t *data = &mutex->prof_data; + mutex_prof_data_t *data = &mutex->prof_data; data->n_lock_ops++; if (data->prev_owner != tsdn) { data->prev_owner = tsdn; @@ -82,10 +82,10 @@ malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { /* Copy the prof data from mutex for processing. */ JEMALLOC_INLINE void -malloc_lock_prof_read(tsdn_t *tsdn, lock_prof_data_t *data, +malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) { - lock_prof_data_t *source = &mutex->prof_data; - /* Can only read with the lock. */ + mutex_prof_data_t *source = &mutex->prof_data; + /* Can only read holding the mutex. */ malloc_mutex_assert_owner(tsdn, mutex); *data = *source; diff --git a/include/jemalloc/internal/mutex_structs.h b/include/jemalloc/internal/mutex_structs.h index ce80e15d..8d6e7eb2 100644 --- a/include/jemalloc/internal/mutex_structs.h +++ b/include/jemalloc/internal/mutex_structs.h @@ -1,20 +1,20 @@ #ifndef JEMALLOC_INTERNAL_MUTEX_STRUCTS_H #define JEMALLOC_INTERNAL_MUTEX_STRUCTS_H -struct lock_prof_data_s { +struct mutex_prof_data_s { /* * Counters touched on the slow path, i.e. when there is lock * contention. We update them once we have the lock. */ - /* Total time (in nano seconds) spent waiting on this lock. */ + /* Total time (in nano seconds) spent waiting on this mutex. */ uint64_t tot_wait_time; /* Max time (in nano seconds) spent on a single lock operation. */ uint64_t max_wait_time; - /* # of times have to wait for this lock (after spinning). */ + /* # of times have to wait for this mutex (after spinning). */ uint64_t n_wait_times; - /* # of times acquired the lock through local spinning. */ + /* # of times acquired the mutex through local spinning. */ uint64_t n_spin_acquired; - /* Max # of threads waiting for the lock at the same time. */ + /* Max # of threads waiting for the mutex at the same time. */ uint32_t max_n_thds; /* Current # of threads waiting on the lock. Atomic synced. */ uint32_t n_waiting_thds; @@ -25,9 +25,9 @@ struct lock_prof_data_s { * the lock) so that we have a higher chance of them being on the same * cacheline. */ - /* # of times the new lock holder is different from the previous one. */ + /* # of times the mutex holder is different than the previous one. */ uint64_t n_owner_switches; - /* Previous lock holder, to facilitate n_owner_switches. */ + /* Previous mutex holder, to facilitate n_owner_switches. */ tsdn_t *prev_owner; /* # of lock() operations in total. */ uint64_t n_lock_ops; @@ -38,13 +38,13 @@ struct malloc_mutex_s { struct { /* * prof_data is defined first to reduce cacheline - * bouncing: the data is not touched by the lock holder + * bouncing: the data is not touched by the mutex holder * during unlocking, while might be modified by - * contenders. Having it before the lock itself could + * contenders. Having it before the mutex itself could * avoid prefetching a modified cacheline (for the * unlocking thread). */ - lock_prof_data_t prof_data; + mutex_prof_data_t prof_data; #ifdef _WIN32 # if _WIN32_WINNT >= 0x0600 SRWLOCK lock; diff --git a/include/jemalloc/internal/mutex_types.h b/include/jemalloc/internal/mutex_types.h index 0d93fe87..257f69ca 100644 --- a/include/jemalloc/internal/mutex_types.h +++ b/include/jemalloc/internal/mutex_types.h @@ -1,7 +1,7 @@ #ifndef JEMALLOC_INTERNAL_MUTEX_TYPES_H #define JEMALLOC_INTERNAL_MUTEX_TYPES_H -typedef struct lock_prof_data_s lock_prof_data_t; +typedef struct mutex_prof_data_s mutex_prof_data_t; typedef struct malloc_mutex_s malloc_mutex_t; #ifdef _WIN32 diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 3c5a21b5..1af1f91b 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -269,6 +269,7 @@ lg_floor lg_prof_sample malloc_cprintf malloc_getcpu +malloc_mutex_prof_data_reset malloc_mutex_assert_not_owner malloc_mutex_assert_owner malloc_mutex_boot diff --git a/include/jemalloc/internal/stats_structs.h b/include/jemalloc/internal/stats_structs.h index 0e33394b..601c8512 100644 --- a/include/jemalloc/internal/stats_structs.h +++ b/include/jemalloc/internal/stats_structs.h @@ -57,7 +57,7 @@ struct malloc_bin_stats_s { /* Current number of slabs in this bin. */ size_t curslabs; - lock_prof_data_t lock_data; + mutex_prof_data_t mutex_data; }; struct malloc_large_stats_s { @@ -124,12 +124,12 @@ struct arena_stats_s { /* Number of bytes cached in tcache associated with this arena. */ atomic_zu_t tcache_bytes; /* Derived. */ - lock_prof_data_t large_mtx_data; - lock_prof_data_t extent_freelist_mtx_data; - lock_prof_data_t extents_cached_mtx_data; - lock_prof_data_t extents_retained_mtx_data; - lock_prof_data_t decay_mtx_data; - lock_prof_data_t tcache_mtx_data; + mutex_prof_data_t large_mtx_data; + mutex_prof_data_t extent_freelist_mtx_data; + mutex_prof_data_t extents_cached_mtx_data; + mutex_prof_data_t extents_retained_mtx_data; + mutex_prof_data_t decay_mtx_data; + mutex_prof_data_t tcache_mtx_data; /* One element for each large size class. */ malloc_large_stats_t lstats[NSIZES - NBINS]; diff --git a/src/arena.c b/src/arena.c index 266c85fb..57b79c52 100644 --- a/src/arena.c +++ b/src/arena.c @@ -292,14 +292,14 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, tbin->ncached * index2size(i)); } } - malloc_lock_prof_read(tsdn, &astats->tcache_mtx_data, + malloc_mutex_prof_read(tsdn, &astats->tcache_mtx_data, &arena->tcache_ql_mtx); malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } #define READ_ARENA_MUTEX_PROF_DATA(mtx, data) \ malloc_mutex_lock(tsdn, &arena->mtx); \ - malloc_lock_prof_read(tsdn, &astats->data, &arena->mtx); \ + malloc_mutex_prof_read(tsdn, &astats->data, &arena->mtx); \ malloc_mutex_unlock(tsdn, &arena->mtx); /* Gather per arena mutex profiling data. */ @@ -317,7 +317,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(tsdn, &bin->lock); - malloc_lock_prof_read(tsdn, &bstats[i].lock_data, &bin->lock); + malloc_mutex_prof_read(tsdn, &bstats[i].mutex_data, &bin->lock); bstats[i].nmalloc += bin->stats.nmalloc; bstats[i].ndalloc += bin->stats.ndalloc; bstats[i].nrequests += bin->stats.nrequests; diff --git a/src/ctl.c b/src/ctl.c index a880c63a..b2b9e7d9 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -13,13 +13,13 @@ static bool ctl_initialized; static ctl_stats_t *ctl_stats; static ctl_arenas_t *ctl_arenas; -const char *global_lock_names[NUM_GLOBAL_PROF_LOCKS] = { +const char *global_mutex_names[NUM_GLOBAL_PROF_MUTEXES] = { "base", "prof", "ctl" }; -const char *arena_lock_names[NUM_ARENA_PROF_LOCKS] = { +const char *arena_mutex_names[NUM_ARENA_PROF_MUTEXES] = { "large", "extent_freelist", "extents_cached", @@ -28,7 +28,7 @@ const char *arena_lock_names[NUM_ARENA_PROF_LOCKS] = { "tcache" }; -const char *lock_counter_names[NUM_LOCK_PROF_COUNTERS] = { +const char *mutex_counter_names[NUM_MUTEX_PROF_COUNTERS] = { "num_ops", "num_wait", "num_spin_acq", @@ -203,7 +203,7 @@ CTL_PROTO(stats_resident) CTL_PROTO(stats_mapped) CTL_PROTO(stats_retained) -#define LOCK_STATS_CTL_PROTO_GEN(n) \ +#define MUTEX_STATS_CTL_PROTO_GEN(n) \ CTL_PROTO(stats_##n##_num_ops) \ CTL_PROTO(stats_##n##_num_wait) \ CTL_PROTO(stats_##n##_num_spin_acq) \ @@ -212,25 +212,27 @@ CTL_PROTO(stats_##n##_total_wait_time) \ CTL_PROTO(stats_##n##_max_wait_time) \ CTL_PROTO(stats_##n##_max_num_thds) -/* Global locks. */ -LOCK_STATS_CTL_PROTO_GEN(locks_base) -LOCK_STATS_CTL_PROTO_GEN(locks_prof) -LOCK_STATS_CTL_PROTO_GEN(locks_ctl) +/* Global mutexes. */ +MUTEX_STATS_CTL_PROTO_GEN(mutexes_base) +MUTEX_STATS_CTL_PROTO_GEN(mutexes_prof) +MUTEX_STATS_CTL_PROTO_GEN(mutexes_ctl) -/* Arena bin locks. */ -LOCK_STATS_CTL_PROTO_GEN(arenas_i_bins_j_lock) +/* Arena bin mutexes. */ +MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex) -#define ARENA_LOCKS_CTL_PROTO_GEN(n) \ - LOCK_STATS_CTL_PROTO_GEN(arenas_i_locks_##n) -/* Per arena locks. */ -ARENA_LOCKS_CTL_PROTO_GEN(large) -ARENA_LOCKS_CTL_PROTO_GEN(extent_freelist) -ARENA_LOCKS_CTL_PROTO_GEN(extents_cached) -ARENA_LOCKS_CTL_PROTO_GEN(extents_retained) -ARENA_LOCKS_CTL_PROTO_GEN(decay) -ARENA_LOCKS_CTL_PROTO_GEN(tcache) -#undef ARENA_LOCKS_CTL_PROTO_GEN -#undef LOCK_STATS_CTL_PROTO_GEN +#define ARENA_MUTEXES_CTL_PROTO_GEN(n) \ + MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##n) +/* Per arena mutexes. */ +ARENA_MUTEXES_CTL_PROTO_GEN(large) +ARENA_MUTEXES_CTL_PROTO_GEN(extent_freelist) +ARENA_MUTEXES_CTL_PROTO_GEN(extents_cached) +ARENA_MUTEXES_CTL_PROTO_GEN(extents_retained) +ARENA_MUTEXES_CTL_PROTO_GEN(decay) +ARENA_MUTEXES_CTL_PROTO_GEN(tcache) +#undef ARENA_MUTEXES_CTL_PROTO_GEN +#undef MUTEX_STATS_CTL_PROTO_GEN + +CTL_PROTO(stats_mutexes_reset) /******************************************************************************/ /* mallctl tree. */ @@ -399,7 +401,7 @@ static const ctl_named_node_t stats_arenas_i_large_node[] = { {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} }; -#define LOCK_PROF_DATA_NODE(prefix) \ +#define MUTEX_PROF_DATA_NODE(prefix) \ static const ctl_named_node_t stats_##prefix##_node[] = { \ {NAME("num_ops"), \ CTL(stats_##prefix##_num_ops)}, \ @@ -418,7 +420,7 @@ static const ctl_named_node_t stats_##prefix##_node[] = { \ /* Note that # of current waiting thread not provided. */ \ }; -LOCK_PROF_DATA_NODE(arenas_i_bins_j_lock) +MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex) static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, @@ -430,7 +432,7 @@ static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, - {NAME("lock"), CHILD(named, stats_arenas_i_bins_j_lock)} + {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)} }; static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { @@ -455,25 +457,25 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { {INDEX(stats_arenas_i_lextents_j)} }; -#define ARENA_LOCK_PROF_DATA_NODE(n) LOCK_PROF_DATA_NODE(arenas_i_locks_##n) +#define ARENA_MUTEX_PROF_DATA_NODE(n) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##n) -ARENA_LOCK_PROF_DATA_NODE(large) -ARENA_LOCK_PROF_DATA_NODE(extent_freelist) -ARENA_LOCK_PROF_DATA_NODE(extents_cached) -ARENA_LOCK_PROF_DATA_NODE(extents_retained) -ARENA_LOCK_PROF_DATA_NODE(decay) -ARENA_LOCK_PROF_DATA_NODE(tcache) +ARENA_MUTEX_PROF_DATA_NODE(large) +ARENA_MUTEX_PROF_DATA_NODE(extent_freelist) +ARENA_MUTEX_PROF_DATA_NODE(extents_cached) +ARENA_MUTEX_PROF_DATA_NODE(extents_retained) +ARENA_MUTEX_PROF_DATA_NODE(decay) +ARENA_MUTEX_PROF_DATA_NODE(tcache) -static const ctl_named_node_t stats_arenas_i_locks_node[] = { - {NAME("large"), CHILD(named, stats_arenas_i_locks_large)}, +static const ctl_named_node_t stats_arenas_i_mutexes_node[] = { + {NAME("large"), CHILD(named, stats_arenas_i_mutexes_large)}, {NAME("extent_freelist"), - CHILD(named, stats_arenas_i_locks_extent_freelist)}, + CHILD(named, stats_arenas_i_mutexes_extent_freelist)}, {NAME("extents_cached"), - CHILD(named, stats_arenas_i_locks_extents_cached)}, + CHILD(named, stats_arenas_i_mutexes_extents_cached)}, {NAME("extents_retained"), - CHILD(named, stats_arenas_i_locks_extents_retained)}, - {NAME("decay"), CHILD(named, stats_arenas_i_locks_decay)}, - {NAME("tcache"), CHILD(named, stats_arenas_i_locks_tcache)} + CHILD(named, stats_arenas_i_mutexes_extents_retained)}, + {NAME("decay"), CHILD(named, stats_arenas_i_mutexes_decay)}, + {NAME("tcache"), CHILD(named, stats_arenas_i_mutexes_tcache)} }; static const ctl_named_node_t stats_arenas_i_node[] = { @@ -500,7 +502,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("large"), CHILD(named, stats_arenas_i_large)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, - {NAME("locks"), CHILD(named, stats_arenas_i_locks)} + {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} }; static const ctl_named_node_t super_stats_arenas_i_node[] = { {NAME(""), CHILD(named, stats_arenas_i)} @@ -510,13 +512,14 @@ static const ctl_indexed_node_t stats_arenas_node[] = { {INDEX(stats_arenas_i)} }; -LOCK_PROF_DATA_NODE(locks_base) -LOCK_PROF_DATA_NODE(locks_prof) -LOCK_PROF_DATA_NODE(locks_ctl) -static const ctl_named_node_t stats_locks_node[] = { - {NAME("base"), CHILD(named, stats_locks_base)}, - {NAME("prof"), CHILD(named, stats_locks_prof)}, - {NAME("ctl"), CHILD(named, stats_locks_ctl)} +MUTEX_PROF_DATA_NODE(mutexes_base) +MUTEX_PROF_DATA_NODE(mutexes_prof) +MUTEX_PROF_DATA_NODE(mutexes_ctl) +static const ctl_named_node_t stats_mutexes_node[] = { + {NAME("base"), CHILD(named, stats_mutexes_base)}, + {NAME("prof"), CHILD(named, stats_mutexes_prof)}, + {NAME("ctl"), CHILD(named, stats_mutexes_ctl)}, + {NAME("reset"), CTL(stats_mutexes_reset)} }; static const ctl_named_node_t stats_node[] = { @@ -526,10 +529,10 @@ static const ctl_named_node_t stats_node[] = { {NAME("resident"), CTL(stats_resident)}, {NAME("mapped"), CTL(stats_mapped)}, {NAME("retained"), CTL(stats_retained)}, - {NAME("locks"), CHILD(named, stats_locks)}, + {NAME("mutexes"), CHILD(named, stats_mutexes)}, {NAME("arenas"), CHILD(indexed, stats_arenas)} }; -#undef LOCK_PROF_DATA_NODE +#undef MUTEX_PROF_DATA_NODE static const ctl_named_node_t root_node[] = { {NAME("version"), CTL(version)}, @@ -768,20 +771,20 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, &astats->astats.decay_muzzy.purged); - malloc_lock_prof_merge(&(sdstats->astats.large_mtx_data), + malloc_mutex_prof_merge(&(sdstats->astats.large_mtx_data), &(astats->astats.large_mtx_data)); - malloc_lock_prof_merge( + malloc_mutex_prof_merge( &(sdstats->astats.extent_freelist_mtx_data), &(astats->astats.extent_freelist_mtx_data)); - malloc_lock_prof_merge( + malloc_mutex_prof_merge( &(sdstats->astats.extents_cached_mtx_data), &(astats->astats.extents_cached_mtx_data)); - malloc_lock_prof_merge( + malloc_mutex_prof_merge( &(sdstats->astats.extents_retained_mtx_data), &(astats->astats.extents_retained_mtx_data)); - malloc_lock_prof_merge(&(sdstats->astats.decay_mtx_data), + malloc_mutex_prof_merge(&(sdstats->astats.decay_mtx_data), &(astats->astats.decay_mtx_data)); - malloc_lock_prof_merge(&(sdstats->astats.tcache_mtx_data), + malloc_mutex_prof_merge(&(sdstats->astats.tcache_mtx_data), &(astats->astats.tcache_mtx_data)); if (!destroyed) { @@ -849,8 +852,8 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, } else { assert(astats->bstats[i].curslabs == 0); } - malloc_lock_prof_merge(&sdstats->bstats[i].lock_data, - &astats->bstats[i].lock_data); + malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data, + &astats->bstats[i].mutex_data); } for (i = 0; i < NSIZES - NBINS; i++) { @@ -956,7 +959,7 @@ ctl_refresh(tsdn_t *tsdn) { #define READ_GLOBAL_MUTEX_PROF_DATA(mtx, data) \ malloc_mutex_lock(tsdn, &mtx); \ - malloc_lock_prof_read(tsdn, &ctl_stats->data, &mtx); \ + malloc_mutex_prof_read(tsdn, &ctl_stats->data, &mtx); \ malloc_mutex_unlock(tsdn, &mtx); READ_GLOBAL_MUTEX_PROF_DATA(b0get()->mtx, base_mtx_data); @@ -964,7 +967,7 @@ ctl_refresh(tsdn_t *tsdn) { READ_GLOBAL_MUTEX_PROF_DATA(bt2gctx_mtx, prof_mtx_data); } /* We own ctl mutex already. */ - malloc_lock_prof_read(tsdn, &ctl_stats->ctl_mtx_data, &ctl_mtx); + malloc_mutex_prof_read(tsdn, &ctl_stats->ctl_mtx_data, &ctl_mtx); #undef READ_GLOBAL_MUTEX_PROF_DATA } ctl_arenas->epoch++; @@ -2454,7 +2457,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, uint64_t) /* Intentional. */ /* Lock profiling related APIs below. */ -#define RO_LOCK_CTL_GEN(n, l) \ +#define RO_MUTEX_CTL_GEN(n, l) \ CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \ l.n_lock_ops, uint64_t) \ CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \ @@ -2470,32 +2473,81 @@ CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ l.max_n_thds, uint64_t) -/* Global lock. */ -#define MTX(mutex) RO_LOCK_CTL_GEN(locks_##mutex, ctl_stats->mutex##_mtx_data) +/* Global mutexes. */ +#define MTX(mutex) \ + RO_MUTEX_CTL_GEN(mutexes_##mutex, ctl_stats->mutex##_mtx_data) GLOBAL_PROF_MUTEXES #undef MTX /* arena->bins[j].lock */ -RO_LOCK_CTL_GEN(arenas_i_bins_j_lock, - arenas_i(mib[2])->astats->bstats[mib[4]].lock_data) +RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, + arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) -/* Per arena locks */ -#define ARENAS_ASTATS_LOCK_CTL_GEN(l, d) \ - RO_LOCK_CTL_GEN(arenas_i_locks_##l, arenas_i(mib[2])->astats->astats.d) +/* Per arena mutexes */ +#define ARENAS_ASTATS_MUTEX_CTL_GEN(l, d) \ + RO_MUTEX_CTL_GEN(arenas_i_mutexes_##l, arenas_i(mib[2])->astats->astats.d) /* arena->large_mtx */ -ARENAS_ASTATS_LOCK_CTL_GEN(large, large_mtx_data) +ARENAS_ASTATS_MUTEX_CTL_GEN(large, large_mtx_data) /* arena->extent_freelist_mtx */ -ARENAS_ASTATS_LOCK_CTL_GEN(extent_freelist, extent_freelist_mtx_data) +ARENAS_ASTATS_MUTEX_CTL_GEN(extent_freelist, extent_freelist_mtx_data) /* arena->extents_cached.mtx */ -ARENAS_ASTATS_LOCK_CTL_GEN(extents_cached, extents_cached_mtx_data) +ARENAS_ASTATS_MUTEX_CTL_GEN(extents_cached, extents_cached_mtx_data) /* arena->extents_retained.mtx */ -ARENAS_ASTATS_LOCK_CTL_GEN(extents_retained, extents_retained_mtx_data) +ARENAS_ASTATS_MUTEX_CTL_GEN(extents_retained, extents_retained_mtx_data) /* arena->decay.mtx */ -ARENAS_ASTATS_LOCK_CTL_GEN(decay, decay_mtx_data) +ARENAS_ASTATS_MUTEX_CTL_GEN(decay, decay_mtx_data) /* arena->tcache_ql_mtx */ -ARENAS_ASTATS_LOCK_CTL_GEN(tcache, tcache_mtx_data) -#undef ARENAS_ASTATS_LOCK_CTL_GEN -#undef RO_LOCK_CTL_GEN +ARENAS_ASTATS_MUTEX_CTL_GEN(tcache, tcache_mtx_data) +#undef ARENAS_ASTATS_MUTEX_CTL_GEN +#undef RO_MUTEX_CTL_GEN + +/* Resets all mutex stats, including global, arena and bin mutexes. */ +static int +stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + if (!config_stats) { + return ENOENT; + } + + tsdn_t *tsdn = tsd_tsdn(tsd); + +#define MUTEX_PROF_RESET(mtx) \ + malloc_mutex_lock(tsdn, &mtx); \ + malloc_mutex_prof_data_reset(tsdn, &mtx); \ + malloc_mutex_unlock(tsdn, &mtx); + + /* Global mutexes: base, prof and ctl. */ + MUTEX_PROF_RESET(b0get()->mtx); + if (config_prof && opt_prof) { + MUTEX_PROF_RESET(bt2gctx_mtx); + } + MUTEX_PROF_RESET(ctl_mtx); + + /* Per arena mutexes. */ + unsigned n = narenas_total_get(); + + for (unsigned i = 0; i < n; i++) { + arena_t *arena = arena_get(tsdn, i, false); + if (!arena) { + continue; + } + MUTEX_PROF_RESET(arena->large_mtx); + MUTEX_PROF_RESET(arena->extent_freelist_mtx); + MUTEX_PROF_RESET(arena->extents_cached.mtx); + MUTEX_PROF_RESET(arena->extents_retained.mtx); + MUTEX_PROF_RESET(arena->decay.mtx); + if (config_tcache) { + MUTEX_PROF_RESET(arena->tcache_ql_mtx); + } + + for (szind_t i = 0; i < NBINS; i++) { + arena_bin_t *bin = &arena->bins[i]; + MUTEX_PROF_RESET(bin->lock); + } + } +#undef MUTEX_PROF_RESET + return 0; +} CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, arenas_i(mib[2])->astats->bstats[mib[4]].nmalloc, uint64_t) diff --git a/src/mutex.c b/src/mutex.c index af6f3c19..82a5fa3e 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -67,7 +67,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void malloc_mutex_lock_slow(malloc_mutex_t *mutex) { - lock_prof_data_t *data = &mutex->prof_data; + mutex_prof_data_t *data = &mutex->prof_data; {//TODO: a smart spin policy if (!malloc_mutex_trylock(mutex)) { @@ -108,15 +108,21 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) { } static void -lock_prof_data_init(lock_prof_data_t *data) { - memset(data, 0, sizeof(lock_prof_data_t)); +mutex_prof_data_init(mutex_prof_data_t *data) { + memset(data, 0, sizeof(mutex_prof_data_t)); data->prev_owner = NULL; } +void +malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex) { + malloc_mutex_assert_owner(tsdn, mutex); + mutex_prof_data_init(&mutex->prof_data); +} + bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank) { - lock_prof_data_init(&mutex->prof_data); + mutex_prof_data_init(&mutex->prof_data); #ifdef _WIN32 # if _WIN32_WINNT >= 0x0600 InitializeSRWLock(&mutex->lock); diff --git a/src/stats.c b/src/stats.c index b6f2124a..ada95e92 100644 --- a/src/stats.c +++ b/src/stats.c @@ -58,20 +58,20 @@ get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) { } static void -gen_lock_ctl_str(char *str, const char *prefix, const char *lock, +gen_mutex_ctl_str(char *str, const char *prefix, const char *mutex, const char *counter) { - sprintf(str, "stats.%s.%s.%s", prefix, lock, counter); + malloc_snprintf(str, 128, "stats.%s.%s.%s", prefix, mutex, counter); } static void -read_arena_bin_lock_stats(unsigned arena_ind, unsigned bin_ind, - uint64_t results[NUM_LOCK_PROF_COUNTERS]) { +read_arena_bin_mutex_stats(unsigned arena_ind, unsigned bin_ind, + uint64_t results[NUM_MUTEX_PROF_COUNTERS]) { char cmd[128]; unsigned i; - for (i = 0; i < NUM_LOCK_PROF_COUNTERS; i++) { - gen_lock_ctl_str(cmd, "arenas.0.bins.0","lock", - lock_counter_names[i]); + for (i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) { + gen_mutex_ctl_str(cmd, "arenas.0.bins.0","mutex", + mutex_counter_names[i]); CTL_M2_M4_GET(cmd, arena_ind, bin_ind, &results[i], uint64_t); } } @@ -147,8 +147,8 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, size_t); if (json) { - uint64_t lock_stats[NUM_LOCK_PROF_COUNTERS]; - read_arena_bin_lock_stats(i, j, lock_stats); + uint64_t mutex_stats[NUM_MUTEX_PROF_COUNTERS]; + read_arena_bin_mutex_stats(i, j, mutex_stats); malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\t{\n" @@ -170,15 +170,15 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\t\t\"nreslabs\": %"FMTu64",\n" "\t\t\t\t\t\t\"curslabs\": %zu,\n" - "\t\t\t\t\t\t\"lock\": {\n", + "\t\t\t\t\t\t\"mutex\": {\n", nreslabs, curslabs); - for (unsigned k = 0; k < NUM_LOCK_PROF_COUNTERS; k++) { + for (unsigned k = 0; k < NUM_MUTEX_PROF_COUNTERS; k++) { malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\t\t\t\"%s\": %"FMTu64"%s\n", - lock_counter_names[k], lock_stats[k], - k == NUM_LOCK_PROF_COUNTERS - 1 ? "" : ","); + mutex_counter_names[k], mutex_stats[k], + k == NUM_MUTEX_PROF_COUNTERS - 1 ? "" : ","); } malloc_cprintf(write_cb, cbopaque, @@ -207,14 +207,14 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, not_reached(); } } - /* Output less info for bin locks to save space. */ + /* Output less info for bin mutexes to save space. */ uint64_t num_ops, num_wait, max_wait; - CTL_M2_M4_GET("stats.arenas.0.bins.0.lock.num_wait", + CTL_M2_M4_GET("stats.arenas.0.bins.0.mutex.num_wait", i, j, &num_wait, uint64_t); CTL_M2_M4_GET( - "stats.arenas.0.bins.0.lock.max_wait_time", i, j, + "stats.arenas.0.bins.0.mutex.max_wait_time", i, j, &max_wait, uint64_t); - CTL_M2_M4_GET("stats.arenas.0.bins.0.lock.num_ops", + CTL_M2_M4_GET("stats.arenas.0.bins.0.mutex.num_ops", i, j, &num_ops, uint64_t); char rate[6]; @@ -325,38 +325,38 @@ stats_arena_lextents_print(void (*write_cb)(void *, const char *), } static void -read_arena_lock_stats(unsigned arena_ind, - uint64_t results[NUM_ARENA_PROF_LOCKS][NUM_LOCK_PROF_COUNTERS]) { +read_arena_mutex_stats(unsigned arena_ind, + uint64_t results[NUM_ARENA_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]) { char cmd[128]; unsigned i, j; - for (i = 0; i < NUM_ARENA_PROF_LOCKS; i++) { - for (j = 0; j < NUM_LOCK_PROF_COUNTERS; j++) { - gen_lock_ctl_str(cmd, "arenas.0.locks", - arena_lock_names[i], lock_counter_names[j]); + for (i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { + for (j = 0; j < NUM_MUTEX_PROF_COUNTERS; j++) { + gen_mutex_ctl_str(cmd, "arenas.0.mutexes", + arena_mutex_names[i], mutex_counter_names[j]); CTL_M2_GET(cmd, arena_ind, &results[i][j], uint64_t); } } } static void -lock_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque, - const char *name, uint64_t stats[NUM_LOCK_PROF_COUNTERS], +mutex_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque, + const char *name, uint64_t stats[NUM_MUTEX_PROF_COUNTERS], const char *json_indent, bool last) { malloc_cprintf(write_cb, cbopaque, "%s\"%s\": {\n", json_indent, name); - for (unsigned i = 0; i < NUM_LOCK_PROF_COUNTERS; i++) { + for (unsigned i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) { malloc_cprintf(write_cb, cbopaque, "%s\t\"%s\": %"FMTu64"%s\n", - json_indent, lock_counter_names[i], stats[i], - i < (NUM_LOCK_PROF_COUNTERS - 1) ? "," : ""); + json_indent, mutex_counter_names[i], stats[i], + i < (NUM_MUTEX_PROF_COUNTERS - 1) ? "," : ""); } malloc_cprintf(write_cb, cbopaque, "%s}%s\n", json_indent, last ? "" : ","); } static void -lock_stats_output(void (*write_cb)(void *, const char *), void *cbopaque, - const char *name, uint64_t stats[NUM_LOCK_PROF_COUNTERS], +mutex_stats_output(void (*write_cb)(void *, const char *), void *cbopaque, + const char *name, uint64_t stats[NUM_MUTEX_PROF_COUNTERS], bool first_mutex) { if (first_mutex) { /* Print title. */ @@ -370,39 +370,39 @@ lock_stats_output(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, ":%*c", (int)(19 - strlen(name)), ' '); - for (unsigned i = 0; i < NUM_LOCK_PROF_COUNTERS; i++) { + for (unsigned i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) { malloc_cprintf(write_cb, cbopaque, " %16"FMTu64, stats[i]); } malloc_cprintf(write_cb, cbopaque, "\n"); } static void -stats_arena_locks_print(void (*write_cb)(void *, const char *), +stats_arena_mutexes_print(void (*write_cb)(void *, const char *), void *cbopaque, bool json, bool json_end, unsigned arena_ind) { - uint64_t lock_stats[NUM_ARENA_PROF_LOCKS][NUM_LOCK_PROF_COUNTERS]; - read_arena_lock_stats(arena_ind, lock_stats); + uint64_t mutex_stats[NUM_ARENA_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]; + read_arena_mutex_stats(arena_ind, mutex_stats); - /* Output lock stats. */ + /* Output mutex stats. */ if (json) { - malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"locks\": {\n"); - for (unsigned i = 0; i < NUM_ARENA_PROF_LOCKS; i++) { - lock_stats_output_json(write_cb, cbopaque, - arena_lock_names[i], lock_stats[i], - "\t\t\t\t\t", (i == NUM_ARENA_PROF_LOCKS - 1)); + malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"mutexes\": {\n"); + for (unsigned i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { + mutex_stats_output_json(write_cb, cbopaque, + arena_mutex_names[i], mutex_stats[i], + "\t\t\t\t\t", (i == NUM_ARENA_PROF_MUTEXES - 1)); } malloc_cprintf(write_cb, cbopaque, "\t\t\t\t}%s\n", json_end ? "" : ","); } else { - for (unsigned i = 0; i < NUM_ARENA_PROF_LOCKS; i++) { - lock_stats_output(write_cb, cbopaque, - arena_lock_names[i], lock_stats[i], i == 0); + for (unsigned i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { + mutex_stats_output(write_cb, cbopaque, + arena_mutex_names[i], mutex_stats[i], i == 0); } } } static void stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, - bool json, unsigned i, bool bins, bool large, bool lock) { + bool json, unsigned i, bool bins, bool large, bool mutex) { unsigned nthreads; const char *dss; ssize_t dirty_decay_time, muzzy_decay_time; @@ -625,14 +625,14 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, if (json) { malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"resident\": %zu%s\n", resident, - (bins || large || lock) ? "," : ""); + (bins || large || mutex) ? "," : ""); } else { malloc_cprintf(write_cb, cbopaque, "resident: %12zu\n", resident); } - if (lock) { - stats_arena_locks_print(write_cb, cbopaque, json, + if (mutex) { + stats_arena_mutexes_print(write_cb, cbopaque, json, !(bins || large), i); } if (bins) { @@ -993,15 +993,16 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, } } -static void read_global_lock_stats( - uint64_t results[NUM_GLOBAL_PROF_LOCKS][NUM_LOCK_PROF_COUNTERS]) { +static void +read_global_mutex_stats( + uint64_t results[NUM_GLOBAL_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]) { char cmd[128]; unsigned i, j; - for (i = 0; i < NUM_GLOBAL_PROF_LOCKS; i++) { - for (j = 0; j < NUM_LOCK_PROF_COUNTERS; j++) { - gen_lock_ctl_str(cmd, "locks", global_lock_names[i], - lock_counter_names[j]); + for (i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { + for (j = 0; j < NUM_MUTEX_PROF_COUNTERS; j++) { + gen_mutex_ctl_str(cmd, "mutexes", global_mutex_names[i], + mutex_counter_names[j]); CTL_GET(cmd, &results[i][j], uint64_t); } } @@ -1010,7 +1011,7 @@ static void read_global_lock_stats( static void stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, bool json, bool merged, bool destroyed, bool unmerged, bool bins, - bool large, bool lock) { + bool large, bool mutex) { size_t allocated, active, metadata, resident, mapped, retained; CTL_GET("stats.allocated", &allocated, size_t); @@ -1020,9 +1021,9 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("stats.mapped", &mapped, size_t); CTL_GET("stats.retained", &retained, size_t); - uint64_t lock_stats[NUM_GLOBAL_PROF_LOCKS][NUM_LOCK_PROF_COUNTERS]; - if (lock) { - read_global_lock_stats(lock_stats); + uint64_t mutex_stats[NUM_GLOBAL_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]; + if (mutex) { + read_global_mutex_stats(mutex_stats); } if (json) { @@ -1041,14 +1042,14 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, "\t\t\t\"mapped\": %zu,\n", mapped); malloc_cprintf(write_cb, cbopaque, "\t\t\t\"retained\": %zu,\n", retained); - if (lock) { + if (mutex) { malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"locks\": {\n"); + "\t\t\t\"mutexes\": {\n"); - for (unsigned i = 0; i < NUM_GLOBAL_PROF_LOCKS; i++) { - lock_stats_output_json(write_cb, cbopaque, - global_lock_names[i], lock_stats[i], - "\t\t\t\t", i == NUM_GLOBAL_PROF_LOCKS - 1); + for (unsigned i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { + mutex_stats_output_json(write_cb, cbopaque, + global_mutex_names[i], mutex_stats[i], + "\t\t\t\t", i == NUM_GLOBAL_PROF_MUTEXES - 1); } malloc_cprintf(write_cb, cbopaque, "\t\t\t}\n"); } @@ -1059,10 +1060,10 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, "Allocated: %zu, active: %zu, metadata: %zu," " resident: %zu, mapped: %zu, retained: %zu\n", allocated, active, metadata, resident, mapped, retained); - if (lock) { - for (unsigned i = 0; i < NUM_GLOBAL_PROF_LOCKS; i++) { - lock_stats_output(write_cb, cbopaque, - global_lock_names[i], lock_stats[i], + if (mutex) { + for (unsigned i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { + mutex_stats_output(write_cb, cbopaque, + global_mutex_names[i], mutex_stats[i], i == 0); } } @@ -1111,7 +1112,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, "\nMerged arenas stats:\n"); } stats_arena_print(write_cb, cbopaque, json, - MALLCTL_ARENAS_ALL, bins, large, lock); + MALLCTL_ARENAS_ALL, bins, large, mutex); if (json) { malloc_cprintf(write_cb, cbopaque, "\t\t\t}%s\n", @@ -1133,7 +1134,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, } stats_arena_print(write_cb, cbopaque, json, MALLCTL_ARENAS_DESTROYED, bins, large, - lock); + mutex); if (json) { malloc_cprintf(write_cb, cbopaque, "\t\t\t}%s\n", unmerged ? "," : @@ -1159,7 +1160,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, } stats_arena_print(write_cb, cbopaque, json, i, bins, - large, lock); + large, mutex); if (json) { malloc_cprintf(write_cb, cbopaque, @@ -1192,7 +1193,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, bool unmerged = config_stats; bool bins = true; bool large = true; - bool lock = true; + bool mutex = true; /* * Refresh stats, in case mallctl() was called by the application. @@ -1243,7 +1244,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, large = false; break; case 'x': - lock = false; + mutex = false; break; default:; } @@ -1264,7 +1265,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, } if (config_stats) { stats_print_helper(write_cb, cbopaque, json, merged, destroyed, - unmerged, bins, large, lock); + unmerged, bins, large, mutex); } if (json) {