diff --git a/include/jemalloc/internal/ctl_externs.h b/include/jemalloc/internal/ctl_externs.h index 3eddba34..17bbba06 100644 --- a/include/jemalloc/internal/ctl_externs.h +++ b/include/jemalloc/internal/ctl_externs.h @@ -4,14 +4,6 @@ /* Maximum ctl tree depth. */ #define CTL_MAX_DEPTH 7 -#define NUM_GLOBAL_PROF_MUTEXES 3 -#define NUM_ARENA_PROF_MUTEXES 8 -#define NUM_MUTEX_PROF_COUNTERS 7 - -extern const char *arena_mutex_names[NUM_ARENA_PROF_MUTEXES]; -extern const char *global_mutex_names[NUM_GLOBAL_PROF_MUTEXES]; -extern const char *mutex_counter_names[NUM_MUTEX_PROF_COUNTERS]; - int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen); int ctl_nametomib(tsdn_t *tsdn, const char *name, size_t *mibp, diff --git a/include/jemalloc/internal/ctl_structs.h b/include/jemalloc/internal/ctl_structs.h index b1ee3555..af0f78b9 100644 --- a/include/jemalloc/internal/ctl_structs.h +++ b/include/jemalloc/internal/ctl_structs.h @@ -42,9 +42,7 @@ struct ctl_stats_s { size_t mapped; size_t retained; -#define MTX(mutex) mutex_prof_data_t mutex##_mtx_data; - GLOBAL_PROF_MUTEXES -#undef MTX + mutex_prof_data_t mutex_prof_data[num_global_prof_mutexes]; }; struct ctl_arena_s { diff --git a/include/jemalloc/internal/ctl_types.h b/include/jemalloc/internal/ctl_types.h index 562418ca..1b5c76f9 100644 --- a/include/jemalloc/internal/ctl_types.h +++ b/include/jemalloc/internal/ctl_types.h @@ -2,9 +2,49 @@ #define JEMALLOC_INTERNAL_CTL_TYPES_H #define GLOBAL_PROF_MUTEXES \ - MTX(base) \ - MTX(ctl) \ - MTX(prof) + OP(base) \ + OP(ctl) \ + OP(prof) + +typedef enum { +#define OP(mtx) global_prof_mutex_##mtx, + GLOBAL_PROF_MUTEXES +#undef OP + num_global_prof_mutexes +} global_prof_mutex_ind_t; + +#define ARENA_PROF_MUTEXES \ + OP(large) \ + OP(extent_freelist) \ + OP(extents_dirty) \ + OP(extents_muzzy) \ + OP(extents_retained) \ + OP(decay_dirty) \ + OP(decay_muzzy) \ + OP(tcache_list) + +typedef enum { +#define OP(mtx) arena_prof_mutex_##mtx, + ARENA_PROF_MUTEXES +#undef OP + num_arena_prof_mutexes +} arena_prof_mutex_ind_t; + +#define MUTEX_PROF_COUNTERS \ + OP(num_ops, uint64_t) \ + OP(num_wait, uint64_t) \ + OP(num_spin_acq, uint64_t) \ + OP(num_owner_switch, uint64_t) \ + OP(total_wait_time, uint64_t) \ + OP(max_wait_time, uint64_t) \ + OP(max_num_thds, uint32_t) + +typedef enum { +#define OP(counter, type) mutex_counter_##counter, + MUTEX_PROF_COUNTERS +#undef OP + num_mutex_prof_counters +} mutex_prof_counter_ind_t; typedef struct ctl_node_s ctl_node_t; typedef struct ctl_named_node_s ctl_named_node_t; diff --git a/include/jemalloc/internal/stats_structs.h b/include/jemalloc/internal/stats_structs.h index 5a9624fb..75a4a783 100644 --- a/include/jemalloc/internal/stats_structs.h +++ b/include/jemalloc/internal/stats_structs.h @@ -124,14 +124,7 @@ struct arena_stats_s { /* Number of bytes cached in tcache associated with this arena. */ atomic_zu_t tcache_bytes; /* Derived. */ - mutex_prof_data_t large_mtx_data; - mutex_prof_data_t extent_freelist_mtx_data; - mutex_prof_data_t extents_dirty_mtx_data; - mutex_prof_data_t extents_muzzy_mtx_data; - mutex_prof_data_t extents_retained_mtx_data; - mutex_prof_data_t decay_dirty_mtx_data; - mutex_prof_data_t decay_muzzy_mtx_data; - mutex_prof_data_t tcache_list_mtx_data; + mutex_prof_data_t mutex_prof_data[num_arena_prof_mutexes]; /* One element for each large size class. */ malloc_large_stats_t lstats[NSIZES - NBINS]; diff --git a/src/arena.c b/src/arena.c index 6bb67338..3383a3b8 100644 --- a/src/arena.c +++ b/src/arena.c @@ -292,28 +292,32 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, tbin->ncached * index2size(i)); } } - malloc_mutex_prof_read(tsdn, &astats->tcache_list_mtx_data, + malloc_mutex_prof_read(tsdn, + &astats->mutex_prof_data[arena_prof_mutex_tcache_list], &arena->tcache_ql_mtx); malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } -#define READ_ARENA_MUTEX_PROF_DATA(mtx, data) \ +#define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \ malloc_mutex_lock(tsdn, &arena->mtx); \ - malloc_mutex_prof_read(tsdn, &astats->data, &arena->mtx); \ + malloc_mutex_prof_read(tsdn, &astats->mutex_prof_data[ind], \ + &arena->mtx); \ malloc_mutex_unlock(tsdn, &arena->mtx); /* Gather per arena mutex profiling data. */ - READ_ARENA_MUTEX_PROF_DATA(large_mtx, large_mtx_data) + READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large); READ_ARENA_MUTEX_PROF_DATA(extent_freelist_mtx, - extent_freelist_mtx_data) + arena_prof_mutex_extent_freelist) READ_ARENA_MUTEX_PROF_DATA(extents_dirty.mtx, - extents_dirty_mtx_data) + arena_prof_mutex_extents_dirty) READ_ARENA_MUTEX_PROF_DATA(extents_muzzy.mtx, - extents_muzzy_mtx_data) + arena_prof_mutex_extents_muzzy) READ_ARENA_MUTEX_PROF_DATA(extents_retained.mtx, - extents_retained_mtx_data) - READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx, decay_dirty_mtx_data) - READ_ARENA_MUTEX_PROF_DATA(decay_muzzy.mtx, decay_muzzy_mtx_data) + arena_prof_mutex_extents_retained) + READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx, + arena_prof_mutex_decay_dirty) + READ_ARENA_MUTEX_PROF_DATA(decay_muzzy.mtx, + arena_prof_mutex_decay_muzzy) #undef READ_ARENA_MUTEX_PROF_DATA for (szind_t i = 0; i < NBINS; i++) { diff --git a/src/ctl.c b/src/ctl.c index 360c6bdc..e2ee36a6 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -13,33 +13,6 @@ static bool ctl_initialized; static ctl_stats_t *ctl_stats; static ctl_arenas_t *ctl_arenas; -const char *global_mutex_names[NUM_GLOBAL_PROF_MUTEXES] = { - "base", - "prof", - "ctl" -}; - -const char *arena_mutex_names[NUM_ARENA_PROF_MUTEXES] = { - "large", - "extent_freelist", - "extents_dirty", - "extents_muzzy", - "extents_retained", - "decay_dirty", - "decay_muzzy", - "tcache_list" -}; - -const char *mutex_counter_names[NUM_MUTEX_PROF_COUNTERS] = { - "num_ops", - "num_wait", - "num_spin_acq", - "num_owner_switch", - "total_wait_time", - "max_wait_time", - "max_num_thds" -}; - /******************************************************************************/ /* Helpers for named and indexed nodes. */ @@ -215,25 +188,17 @@ CTL_PROTO(stats_##n##_max_wait_time) \ CTL_PROTO(stats_##n##_max_num_thds) /* Global mutexes. */ -MUTEX_STATS_CTL_PROTO_GEN(mutexes_base) -MUTEX_STATS_CTL_PROTO_GEN(mutexes_prof) -MUTEX_STATS_CTL_PROTO_GEN(mutexes_ctl) +#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx) +GLOBAL_PROF_MUTEXES +#undef OP + +/* Per arena mutexes. */ +#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx) +ARENA_PROF_MUTEXES +#undef OP /* Arena bin mutexes. */ MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex) - -#define ARENA_MUTEXES_CTL_PROTO_GEN(n) \ - MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##n) -/* Per arena mutexes. */ -ARENA_MUTEXES_CTL_PROTO_GEN(large) -ARENA_MUTEXES_CTL_PROTO_GEN(extent_freelist) -ARENA_MUTEXES_CTL_PROTO_GEN(extents_dirty) -ARENA_MUTEXES_CTL_PROTO_GEN(extents_muzzy) -ARENA_MUTEXES_CTL_PROTO_GEN(extents_retained) -ARENA_MUTEXES_CTL_PROTO_GEN(decay_dirty) -ARENA_MUTEXES_CTL_PROTO_GEN(decay_muzzy) -ARENA_MUTEXES_CTL_PROTO_GEN(tcache_list) -#undef ARENA_MUTEXES_CTL_PROTO_GEN #undef MUTEX_STATS_CTL_PROTO_GEN CTL_PROTO(stats_mutexes_reset) @@ -461,34 +426,14 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { {INDEX(stats_arenas_i_lextents_j)} }; -#define ARENA_MUTEX_PROF_DATA_NODE(n) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##n) - -ARENA_MUTEX_PROF_DATA_NODE(large) -ARENA_MUTEX_PROF_DATA_NODE(extent_freelist) -ARENA_MUTEX_PROF_DATA_NODE(extents_dirty) -ARENA_MUTEX_PROF_DATA_NODE(extents_muzzy) -ARENA_MUTEX_PROF_DATA_NODE(extents_retained) -ARENA_MUTEX_PROF_DATA_NODE(decay_dirty) -ARENA_MUTEX_PROF_DATA_NODE(decay_muzzy) -ARENA_MUTEX_PROF_DATA_NODE(tcache_list) +#define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) +ARENA_PROF_MUTEXES +#undef OP static const ctl_named_node_t stats_arenas_i_mutexes_node[] = { - {NAME("large"), - CHILD(named, stats_arenas_i_mutexes_large)}, - {NAME("extent_freelist"), - CHILD(named, stats_arenas_i_mutexes_extent_freelist)}, - {NAME("extents_dirty"), - CHILD(named, stats_arenas_i_mutexes_extents_dirty)}, - {NAME("extents_muzzy"), - CHILD(named, stats_arenas_i_mutexes_extents_muzzy)}, - {NAME("extents_retained"), - CHILD(named, stats_arenas_i_mutexes_extents_retained)}, - {NAME("decay_dirty"), - CHILD(named, stats_arenas_i_mutexes_decay_dirty)}, - {NAME("decay_muzzy"), - CHILD(named, stats_arenas_i_mutexes_decay_muzzy)}, - {NAME("tcache_list"), - CHILD(named, stats_arenas_i_mutexes_tcache_list)} +#define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)}, +ARENA_PROF_MUTEXES +#undef OP }; static const ctl_named_node_t stats_arenas_i_node[] = { @@ -525,15 +470,17 @@ static const ctl_indexed_node_t stats_arenas_node[] = { {INDEX(stats_arenas_i)} }; -MUTEX_PROF_DATA_NODE(mutexes_base) -MUTEX_PROF_DATA_NODE(mutexes_prof) -MUTEX_PROF_DATA_NODE(mutexes_ctl) +#define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx) +GLOBAL_PROF_MUTEXES +#undef OP + static const ctl_named_node_t stats_mutexes_node[] = { - {NAME("base"), CHILD(named, stats_mutexes_base)}, - {NAME("prof"), CHILD(named, stats_mutexes_prof)}, - {NAME("ctl"), CHILD(named, stats_mutexes_ctl)}, +#define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)}, +GLOBAL_PROF_MUTEXES +#undef OP {NAME("reset"), CTL(stats_mutexes_reset)} }; +#undef MUTEX_PROF_DATA_NODE static const ctl_named_node_t stats_node[] = { {NAME("allocated"), CTL(stats_allocated)}, @@ -545,7 +492,6 @@ static const ctl_named_node_t stats_node[] = { {NAME("mutexes"), CHILD(named, stats_mutexes)}, {NAME("arenas"), CHILD(indexed, stats_arenas)} }; -#undef MUTEX_PROF_DATA_NODE static const ctl_named_node_t root_node[] = { {NAME("version"), CTL(version)}, @@ -784,27 +730,13 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, &astats->astats.decay_muzzy.purged); - malloc_mutex_prof_merge(&(sdstats->astats.large_mtx_data), - &(astats->astats.large_mtx_data)); - malloc_mutex_prof_merge( - &(sdstats->astats.extent_freelist_mtx_data), - &(astats->astats.extent_freelist_mtx_data)); - malloc_mutex_prof_merge( - &(sdstats->astats.extents_dirty_mtx_data), - &(astats->astats.extents_dirty_mtx_data)); - malloc_mutex_prof_merge( - &(sdstats->astats.extents_muzzy_mtx_data), - &(astats->astats.extents_muzzy_mtx_data)); - malloc_mutex_prof_merge( - &(sdstats->astats.extents_retained_mtx_data), - &(astats->astats.extents_retained_mtx_data)); - malloc_mutex_prof_merge(&(sdstats->astats.decay_dirty_mtx_data), - &(astats->astats.decay_dirty_mtx_data)); - malloc_mutex_prof_merge(&(sdstats->astats.decay_muzzy_mtx_data), - &(astats->astats.decay_muzzy_mtx_data)); - malloc_mutex_prof_merge(&(sdstats->astats.tcache_list_mtx_data), - &(astats->astats.tcache_list_mtx_data)); - +#define OP(mtx) malloc_mutex_prof_merge( \ + &(sdstats->astats.mutex_prof_data[ \ + arena_prof_mutex_##mtx]), \ + &(astats->astats.mutex_prof_data[ \ + arena_prof_mutex_##mtx])); +ARENA_PROF_MUTEXES +#undef OP if (!destroyed) { accum_atomic_zu(&sdstats->astats.base, &astats->astats.base); @@ -975,17 +907,21 @@ ctl_refresh(tsdn_t *tsdn) { ctl_stats->retained = atomic_load_zu( &ctl_sarena->astats->astats.retained, ATOMIC_RELAXED); -#define READ_GLOBAL_MUTEX_PROF_DATA(mtx, data) \ +#define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \ malloc_mutex_lock(tsdn, &mtx); \ - malloc_mutex_prof_read(tsdn, &ctl_stats->data, &mtx); \ + malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \ malloc_mutex_unlock(tsdn, &mtx); - READ_GLOBAL_MUTEX_PROF_DATA(b0get()->mtx, base_mtx_data); + READ_GLOBAL_MUTEX_PROF_DATA(global_prof_mutex_base, + b0get()->mtx); if (config_prof && opt_prof) { - READ_GLOBAL_MUTEX_PROF_DATA(bt2gctx_mtx, prof_mtx_data); + READ_GLOBAL_MUTEX_PROF_DATA(global_prof_mutex_prof, + bt2gctx_mtx); } /* We own ctl mutex already. */ - malloc_mutex_prof_read(tsdn, &ctl_stats->ctl_mtx_data, &ctl_mtx); + malloc_mutex_prof_read(tsdn, + &ctl_stats->mutex_prof_data[global_prof_mutex_ctl], + &ctl_mtx); #undef READ_GLOBAL_MUTEX_PROF_DATA } ctl_arenas->epoch++; @@ -2489,29 +2425,24 @@ CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \ CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ nstime_ns(&l.max_wait_time), uint64_t) \ CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ - l.max_n_thds, uint64_t) + l.max_n_thds, uint32_t) /* Global mutexes. */ -#define MTX(mutex) \ - RO_MUTEX_CTL_GEN(mutexes_##mutex, ctl_stats->mutex##_mtx_data) +#define OP(mtx) \ + RO_MUTEX_CTL_GEN(mutexes_##mtx, \ + ctl_stats->mutex_prof_data[global_prof_mutex_##mtx]) GLOBAL_PROF_MUTEXES -#undef MTX +#undef OP + +/* Per arena mutexes */ +#define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \ + arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx]) +ARENA_PROF_MUTEXES +#undef OP /* tcache bin mutex */ RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) -/* Per arena mutexes */ -#define ARENAS_ASTATS_MUTEX_CTL_GEN(l, d) \ - RO_MUTEX_CTL_GEN(arenas_i_mutexes_##l, arenas_i(mib[2])->astats->astats.d) -ARENAS_ASTATS_MUTEX_CTL_GEN(large, large_mtx_data) -ARENAS_ASTATS_MUTEX_CTL_GEN(extent_freelist, extent_freelist_mtx_data) -ARENAS_ASTATS_MUTEX_CTL_GEN(extents_dirty, extents_dirty_mtx_data) -ARENAS_ASTATS_MUTEX_CTL_GEN(extents_muzzy, extents_muzzy_mtx_data) -ARENAS_ASTATS_MUTEX_CTL_GEN(extents_retained, extents_retained_mtx_data) -ARENAS_ASTATS_MUTEX_CTL_GEN(decay_dirty, decay_dirty_mtx_data) -ARENAS_ASTATS_MUTEX_CTL_GEN(decay_muzzy, decay_muzzy_mtx_data) -ARENAS_ASTATS_MUTEX_CTL_GEN(tcache_list, tcache_list_mtx_data) -#undef ARENAS_ASTATS_MUTEX_CTL_GEN #undef RO_MUTEX_CTL_GEN /* Resets all mutex stats, including global, arena and bin mutexes. */ diff --git a/src/stats.c b/src/stats.c index ada95e92..aa7ca507 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1,6 +1,18 @@ #define JEMALLOC_STATS_C_ #include "jemalloc/internal/jemalloc_internal.h" +const char *global_mutex_names[num_global_prof_mutexes] = { +#define OP(mtx) #mtx, + GLOBAL_PROF_MUTEXES +#undef OP +}; + +const char *arena_mutex_names[num_arena_prof_mutexes] = { +#define OP(mtx) #mtx, + ARENA_PROF_MUTEXES +#undef OP +}; + #define CTL_GET(n, v, t) do { \ size_t sz = sizeof(t); \ xmallctl(n, (void *)v, &sz, NULL, 0); \ @@ -57,28 +69,49 @@ get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) { return false; } +#define MUTEX_CTL_STR_MAX_LENGTH 128 static void -gen_mutex_ctl_str(char *str, const char *prefix, const char *mutex, - const char *counter) { - malloc_snprintf(str, 128, "stats.%s.%s.%s", prefix, mutex, counter); +gen_mutex_ctl_str(char *str, size_t buf_len, const char *prefix, + const char *mutex, const char *counter) { + malloc_snprintf(str, buf_len, "stats.%s.%s.%s", prefix, mutex, counter); } static void read_arena_bin_mutex_stats(unsigned arena_ind, unsigned bin_ind, - uint64_t results[NUM_MUTEX_PROF_COUNTERS]) { - char cmd[128]; + uint64_t results[num_mutex_prof_counters]) { + char cmd[MUTEX_CTL_STR_MAX_LENGTH]; +#define OP(c, t) \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.bins.0","mutex", #c); \ + CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ + (t *)&results[mutex_counter_##c], t); +MUTEX_PROF_COUNTERS +#undef OP +} - unsigned i; - for (i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) { - gen_mutex_ctl_str(cmd, "arenas.0.bins.0","mutex", - mutex_counter_names[i]); - CTL_M2_M4_GET(cmd, arena_ind, bin_ind, &results[i], uint64_t); - } +static void +mutex_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque, + const char *name, uint64_t stats[num_mutex_prof_counters], + const char *json_indent, bool last) { + malloc_cprintf(write_cb, cbopaque, "%s\"%s\": {\n", json_indent, name); + + mutex_prof_counter_ind_t k = 0; + char *fmt_str[2] = {"%s\t\"%s\": %"FMTu32"%s\n", + "%s\t\"%s\": %"FMTu64"%s\n"}; +#define OP(c, t) \ + malloc_cprintf(write_cb, cbopaque, \ + fmt_str[sizeof(t) / sizeof(uint32_t) - 1], \ + json_indent, #c, (t)stats[mutex_counter_##c], \ + (++k == num_mutex_prof_counters) ? "" : ","); +MUTEX_PROF_COUNTERS +#undef OP + malloc_cprintf(write_cb, cbopaque, "%s}%s\n", json_indent, + last ? "" : ","); } static void stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, - bool json, bool large, unsigned i) { + bool json, bool large, bool mutex, unsigned i) { size_t page; bool in_gap, in_gap_prev; unsigned nbins, j; @@ -147,9 +180,6 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, size_t); if (json) { - uint64_t mutex_stats[NUM_MUTEX_PROF_COUNTERS]; - read_arena_bin_mutex_stats(i, j, mutex_stats); - malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\t{\n" "\t\t\t\t\t\t\"nmalloc\": %"FMTu64",\n" @@ -169,20 +199,16 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\t\t\"nreslabs\": %"FMTu64",\n" - "\t\t\t\t\t\t\"curslabs\": %zu,\n" - "\t\t\t\t\t\t\"mutex\": {\n", - nreslabs, - curslabs); + "\t\t\t\t\t\t\"curslabs\": %zu%s\n", + nreslabs, curslabs, mutex ? "," : ""); - for (unsigned k = 0; k < NUM_MUTEX_PROF_COUNTERS; k++) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\t\t\"%s\": %"FMTu64"%s\n", - mutex_counter_names[k], mutex_stats[k], - k == NUM_MUTEX_PROF_COUNTERS - 1 ? "" : ","); + if (mutex) { + uint64_t mutex_stats[num_mutex_prof_counters]; + read_arena_bin_mutex_stats(i, j, mutex_stats); + mutex_stats_output_json(write_cb, cbopaque, + "mutex", mutex_stats, "\t\t\t\t\t\t", true); } - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\t}\n" "\t\t\t\t\t}%s\n", (j + 1 < nbins) ? "," : ""); } else if (!in_gap) { @@ -326,74 +352,79 @@ stats_arena_lextents_print(void (*write_cb)(void *, const char *), static void read_arena_mutex_stats(unsigned arena_ind, - uint64_t results[NUM_ARENA_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]) { - char cmd[128]; + uint64_t results[num_arena_prof_mutexes][num_mutex_prof_counters]) { + char cmd[MUTEX_CTL_STR_MAX_LENGTH]; - unsigned i, j; - for (i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { - for (j = 0; j < NUM_MUTEX_PROF_COUNTERS; j++) { - gen_mutex_ctl_str(cmd, "arenas.0.mutexes", - arena_mutex_names[i], mutex_counter_names[j]); - CTL_M2_GET(cmd, arena_ind, &results[i][j], uint64_t); - } + arena_prof_mutex_ind_t i; + for (i = 0; i < num_arena_prof_mutexes; i++) { +#define OP(c, t) \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.mutexes", arena_mutex_names[i], #c); \ + CTL_M2_GET(cmd, arena_ind, \ + (t *)&results[i][mutex_counter_##c], t); +MUTEX_PROF_COUNTERS +#undef OP } } -static void -mutex_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque, - const char *name, uint64_t stats[NUM_MUTEX_PROF_COUNTERS], - const char *json_indent, bool last) { - - malloc_cprintf(write_cb, cbopaque, "%s\"%s\": {\n", json_indent, name); - for (unsigned i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) { - malloc_cprintf(write_cb, cbopaque, "%s\t\"%s\": %"FMTu64"%s\n", - json_indent, mutex_counter_names[i], stats[i], - i < (NUM_MUTEX_PROF_COUNTERS - 1) ? "," : ""); - } - malloc_cprintf(write_cb, cbopaque, "%s}%s\n", json_indent, - last ? "" : ","); -} - static void mutex_stats_output(void (*write_cb)(void *, const char *), void *cbopaque, - const char *name, uint64_t stats[NUM_MUTEX_PROF_COUNTERS], + const char *name, uint64_t stats[num_mutex_prof_counters], bool first_mutex) { if (first_mutex) { /* Print title. */ malloc_cprintf(write_cb, cbopaque, - " n_lock_ops n_waiting" - " n_spin_acq n_owner_switch total_wait_ns" - " max_wait_ns max_n_wait_thds\n"); + " n_lock_ops n_waiting" + " n_spin_acq n_owner_switch total_wait_ns" + " max_wait_ns max_n_thds\n"); } malloc_cprintf(write_cb, cbopaque, "%s", name); malloc_cprintf(write_cb, cbopaque, ":%*c", - (int)(19 - strlen(name)), ' '); + (int)(20 - strlen(name)), ' '); - for (unsigned i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) { - malloc_cprintf(write_cb, cbopaque, " %16"FMTu64, stats[i]); - } + char *fmt_str[2] = {"%12"FMTu32, "%16"FMTu64}; +#define OP(c, t) \ + malloc_cprintf(write_cb, cbopaque, \ + fmt_str[sizeof(t) / sizeof(uint32_t) - 1], \ + (t)stats[mutex_counter_##c]); +MUTEX_PROF_COUNTERS +#undef OP malloc_cprintf(write_cb, cbopaque, "\n"); } static void stats_arena_mutexes_print(void (*write_cb)(void *, const char *), void *cbopaque, bool json, bool json_end, unsigned arena_ind) { - uint64_t mutex_stats[NUM_ARENA_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]; + uint64_t mutex_stats[num_arena_prof_mutexes][num_mutex_prof_counters]; read_arena_mutex_stats(arena_ind, mutex_stats); /* Output mutex stats. */ if (json) { malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"mutexes\": {\n"); - for (unsigned i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { + arena_prof_mutex_ind_t i, last_mutex; + last_mutex = num_arena_prof_mutexes - 1; + if (!config_tcache) { + last_mutex--; + } + for (i = 0; i < num_arena_prof_mutexes; i++) { + if (!config_tcache && + i == arena_prof_mutex_tcache_list) { + continue; + } mutex_stats_output_json(write_cb, cbopaque, arena_mutex_names[i], mutex_stats[i], - "\t\t\t\t\t", (i == NUM_ARENA_PROF_MUTEXES - 1)); + "\t\t\t\t\t", (i == last_mutex)); } malloc_cprintf(write_cb, cbopaque, "\t\t\t\t}%s\n", json_end ? "" : ","); } else { - for (unsigned i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { + arena_prof_mutex_ind_t i; + for (i = 0; i < num_arena_prof_mutexes; i++) { + if (!config_tcache && + i == arena_prof_mutex_tcache_list) { + continue; + } mutex_stats_output(write_cb, cbopaque, arena_mutex_names[i], mutex_stats[i], i == 0); } @@ -636,7 +667,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, !(bins || large), i); } if (bins) { - stats_arena_bins_print(write_cb, cbopaque, json, large, i); + stats_arena_bins_print(write_cb, cbopaque, json, large, mutex, + i); } if (large) { stats_arena_lextents_print(write_cb, cbopaque, json, i); @@ -995,16 +1027,17 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, static void read_global_mutex_stats( - uint64_t results[NUM_GLOBAL_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]) { - char cmd[128]; + uint64_t results[num_global_prof_mutexes][num_mutex_prof_counters]) { + char cmd[MUTEX_CTL_STR_MAX_LENGTH]; - unsigned i, j; - for (i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { - for (j = 0; j < NUM_MUTEX_PROF_COUNTERS; j++) { - gen_mutex_ctl_str(cmd, "mutexes", global_mutex_names[i], - mutex_counter_names[j]); - CTL_GET(cmd, &results[i][j], uint64_t); - } + global_prof_mutex_ind_t i; + for (i = 0; i < num_global_prof_mutexes; i++) { +#define OP(c, t) \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "mutexes", global_mutex_names[i], #c); \ + CTL_GET(cmd, (t *)&results[i][mutex_counter_##c], t); +MUTEX_PROF_COUNTERS +#undef OP } } @@ -1021,7 +1054,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("stats.mapped", &mapped, size_t); CTL_GET("stats.retained", &retained, size_t); - uint64_t mutex_stats[NUM_GLOBAL_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]; + uint64_t mutex_stats[num_global_prof_mutexes][num_mutex_prof_counters]; if (mutex) { read_global_mutex_stats(mutex_stats); } @@ -1041,15 +1074,16 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "\t\t\t\"mapped\": %zu,\n", mapped); malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"retained\": %zu,\n", retained); + "\t\t\t\"retained\": %zu%s\n", retained, mutex ? "," : ""); if (mutex) { malloc_cprintf(write_cb, cbopaque, "\t\t\t\"mutexes\": {\n"); - - for (unsigned i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { + global_prof_mutex_ind_t i; + for (i = 0; i < num_global_prof_mutexes; i++) { mutex_stats_output_json(write_cb, cbopaque, global_mutex_names[i], mutex_stats[i], - "\t\t\t\t", i == NUM_GLOBAL_PROF_MUTEXES - 1); + "\t\t\t\t", + i == num_global_prof_mutexes - 1); } malloc_cprintf(write_cb, cbopaque, "\t\t\t}\n"); } @@ -1061,7 +1095,8 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, " resident: %zu, mapped: %zu, retained: %zu\n", allocated, active, metadata, resident, mapped, retained); if (mutex) { - for (unsigned i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { + global_prof_mutex_ind_t i; + for (i = 0; i < num_global_prof_mutexes; i++) { mutex_stats_output(write_cb, cbopaque, global_mutex_names[i], mutex_stats[i], i == 0);