project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor mutex profiling code with x-macros.

Browse Source
This commit is contained in:
Qi Wang 2017-03-21 11:56:38 -07:00 committed by Qi Wang
parent f6698ec1e6
commit d3fde1c124
7 changed files with 221 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/jemalloc/internal/ctl_externs.h
View File

@ -4,14 +4,6 @@
/* Maximum ctl tree depth. */ /* Maximum ctl tree depth. */
#define CTL_MAX_DEPTH 7 #define CTL_MAX_DEPTH 7
#define NUM_GLOBAL_PROF_MUTEXES 3
#define NUM_ARENA_PROF_MUTEXES 8
#define NUM_MUTEX_PROF_COUNTERS 7
extern const char *arena_mutex_names[NUM_ARENA_PROF_MUTEXES];
extern const char *global_mutex_names[NUM_GLOBAL_PROF_MUTEXES];
extern const char *mutex_counter_names[NUM_MUTEX_PROF_COUNTERS];
int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp, int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
void *newp, size_t newlen); void *newp, size_t newlen);
int ctl_nametomib(tsdn_t *tsdn, const char *name, size_t *mibp, int ctl_nametomib(tsdn_t *tsdn, const char *name, size_t *mibp,

4
include/jemalloc/internal/ctl_structs.h
View File

@ -42,9 +42,7 @@ struct ctl_stats_s {
size_t mapped; size_t mapped;
size_t retained; size_t retained;
#define MTX(mutex) mutex_prof_data_t mutex##_mtx_data; mutex_prof_data_t mutex_prof_data[num_global_prof_mutexes];
GLOBAL_PROF_MUTEXES
#undef MTX
}; };
struct ctl_arena_s { struct ctl_arena_s {

46
include/jemalloc/internal/ctl_types.h
View File

@ -2,9 +2,49 @@
#define JEMALLOC_INTERNAL_CTL_TYPES_H #define JEMALLOC_INTERNAL_CTL_TYPES_H
#define GLOBAL_PROF_MUTEXES \ #define GLOBAL_PROF_MUTEXES \
MTX(base) \ OP(base) \
MTX(ctl) \ OP(ctl) \
MTX(prof) OP(prof)
typedef enum {
#define OP(mtx) global_prof_mutex_##mtx,
GLOBAL_PROF_MUTEXES
#undef OP
num_global_prof_mutexes
} global_prof_mutex_ind_t;
#define ARENA_PROF_MUTEXES \
OP(large) \
OP(extent_freelist) \
OP(extents_dirty) \
OP(extents_muzzy) \
OP(extents_retained) \
OP(decay_dirty) \
OP(decay_muzzy) \
OP(tcache_list)
typedef enum {
#define OP(mtx) arena_prof_mutex_##mtx,
ARENA_PROF_MUTEXES
#undef OP
num_arena_prof_mutexes
} arena_prof_mutex_ind_t;
#define MUTEX_PROF_COUNTERS \
OP(num_ops, uint64_t) \
OP(num_wait, uint64_t) \
OP(num_spin_acq, uint64_t) \
OP(num_owner_switch, uint64_t) \
OP(total_wait_time, uint64_t) \
OP(max_wait_time, uint64_t) \
OP(max_num_thds, uint32_t)
typedef enum {
#define OP(counter, type) mutex_counter_##counter,
MUTEX_PROF_COUNTERS
#undef OP
num_mutex_prof_counters
} mutex_prof_counter_ind_t;
typedef struct ctl_node_s ctl_node_t; typedef struct ctl_node_s ctl_node_t;
typedef struct ctl_named_node_s ctl_named_node_t; typedef struct ctl_named_node_s ctl_named_node_t;

9
include/jemalloc/internal/stats_structs.h
View File

@ -124,14 +124,7 @@ struct arena_stats_s {
/* Number of bytes cached in tcache associated with this arena. */ /* Number of bytes cached in tcache associated with this arena. */
atomic_zu_t tcache_bytes; /* Derived. */ atomic_zu_t tcache_bytes; /* Derived. */
mutex_prof_data_t large_mtx_data; mutex_prof_data_t mutex_prof_data[num_arena_prof_mutexes];
mutex_prof_data_t extent_freelist_mtx_data;
mutex_prof_data_t extents_dirty_mtx_data;
mutex_prof_data_t extents_muzzy_mtx_data;
mutex_prof_data_t extents_retained_mtx_data;
mutex_prof_data_t decay_dirty_mtx_data;
mutex_prof_data_t decay_muzzy_mtx_data;
mutex_prof_data_t tcache_list_mtx_data;
/* One element for each large size class. */ /* One element for each large size class. */
malloc_large_stats_t lstats[NSIZES - NBINS]; malloc_large_stats_t lstats[NSIZES - NBINS];

24
src/arena.c
View File

@ -292,28 +292,32 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
tbin->ncached * index2size(i)); tbin->ncached * index2size(i));
} }
} }
malloc_mutex_prof_read(tsdn, &astats->tcache_list_mtx_data, malloc_mutex_prof_read(tsdn,
&astats->mutex_prof_data[arena_prof_mutex_tcache_list],
&arena->tcache_ql_mtx); &arena->tcache_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
} }
#define READ_ARENA_MUTEX_PROF_DATA(mtx, data) \ #define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \
malloc_mutex_lock(tsdn, &arena->mtx); \ malloc_mutex_lock(tsdn, &arena->mtx); \
malloc_mutex_prof_read(tsdn, &astats->data, &arena->mtx); \ malloc_mutex_prof_read(tsdn, &astats->mutex_prof_data[ind], \
&arena->mtx); \
malloc_mutex_unlock(tsdn, &arena->mtx); malloc_mutex_unlock(tsdn, &arena->mtx);
/* Gather per arena mutex profiling data. */ /* Gather per arena mutex profiling data. */
READ_ARENA_MUTEX_PROF_DATA(large_mtx, large_mtx_data) READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
READ_ARENA_MUTEX_PROF_DATA(extent_freelist_mtx, READ_ARENA_MUTEX_PROF_DATA(extent_freelist_mtx,
extent_freelist_mtx_data) arena_prof_mutex_extent_freelist)
READ_ARENA_MUTEX_PROF_DATA(extents_dirty.mtx, READ_ARENA_MUTEX_PROF_DATA(extents_dirty.mtx,
extents_dirty_mtx_data) arena_prof_mutex_extents_dirty)
READ_ARENA_MUTEX_PROF_DATA(extents_muzzy.mtx, READ_ARENA_MUTEX_PROF_DATA(extents_muzzy.mtx,
extents_muzzy_mtx_data) arena_prof_mutex_extents_muzzy)
READ_ARENA_MUTEX_PROF_DATA(extents_retained.mtx, READ_ARENA_MUTEX_PROF_DATA(extents_retained.mtx,
extents_retained_mtx_data) arena_prof_mutex_extents_retained)
READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx, decay_dirty_mtx_data) READ_ARENA_MUTEX_PROF_DATA(decay_dirty.mtx,
READ_ARENA_MUTEX_PROF_DATA(decay_muzzy.mtx, decay_muzzy_mtx_data) arena_prof_mutex_decay_dirty)
READ_ARENA_MUTEX_PROF_DATA(decay_muzzy.mtx,
arena_prof_mutex_decay_muzzy)
#undef READ_ARENA_MUTEX_PROF_DATA #undef READ_ARENA_MUTEX_PROF_DATA
for (szind_t i = 0; i < NBINS; i++) { for (szind_t i = 0; i < NBINS; i++) {

167
src/ctl.c
View File

@ -13,33 +13,6 @@ static bool ctl_initialized;
static ctl_stats_t *ctl_stats; static ctl_stats_t *ctl_stats;
static ctl_arenas_t *ctl_arenas; static ctl_arenas_t *ctl_arenas;
const char *global_mutex_names[NUM_GLOBAL_PROF_MUTEXES] = {
"base",
"prof",
"ctl"
};
const char *arena_mutex_names[NUM_ARENA_PROF_MUTEXES] = {
"large",
"extent_freelist",
"extents_dirty",
"extents_muzzy",
"extents_retained",
"decay_dirty",
"decay_muzzy",
"tcache_list"
};
const char *mutex_counter_names[NUM_MUTEX_PROF_COUNTERS] = {
"num_ops",
"num_wait",
"num_spin_acq",
"num_owner_switch",
"total_wait_time",
"max_wait_time",
"max_num_thds"
};
/******************************************************************************/ /******************************************************************************/
/* Helpers for named and indexed nodes. */ /* Helpers for named and indexed nodes. */
@ -215,25 +188,17 @@ CTL_PROTO(stats_##n##_max_wait_time) \
CTL_PROTO(stats_##n##_max_num_thds) CTL_PROTO(stats_##n##_max_num_thds)
/* Global mutexes. */ /* Global mutexes. */
MUTEX_STATS_CTL_PROTO_GEN(mutexes_base) #define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx)
MUTEX_STATS_CTL_PROTO_GEN(mutexes_prof) GLOBAL_PROF_MUTEXES
MUTEX_STATS_CTL_PROTO_GEN(mutexes_ctl) #undef OP
/* Per arena mutexes. */
#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx)
ARENA_PROF_MUTEXES
#undef OP
/* Arena bin mutexes. */ /* Arena bin mutexes. */
MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex)
#define ARENA_MUTEXES_CTL_PROTO_GEN(n) \
MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##n)
/* Per arena mutexes. */
ARENA_MUTEXES_CTL_PROTO_GEN(large)
ARENA_MUTEXES_CTL_PROTO_GEN(extent_freelist)
ARENA_MUTEXES_CTL_PROTO_GEN(extents_dirty)
ARENA_MUTEXES_CTL_PROTO_GEN(extents_muzzy)
ARENA_MUTEXES_CTL_PROTO_GEN(extents_retained)
ARENA_MUTEXES_CTL_PROTO_GEN(decay_dirty)
ARENA_MUTEXES_CTL_PROTO_GEN(decay_muzzy)
ARENA_MUTEXES_CTL_PROTO_GEN(tcache_list)
#undef ARENA_MUTEXES_CTL_PROTO_GEN
#undef MUTEX_STATS_CTL_PROTO_GEN #undef MUTEX_STATS_CTL_PROTO_GEN
CTL_PROTO(stats_mutexes_reset) CTL_PROTO(stats_mutexes_reset)
@ -461,34 +426,14 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
{INDEX(stats_arenas_i_lextents_j)} {INDEX(stats_arenas_i_lextents_j)}
}; };
#define ARENA_MUTEX_PROF_DATA_NODE(n) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##n) #define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx)
ARENA_PROF_MUTEXES
ARENA_MUTEX_PROF_DATA_NODE(large) #undef OP
ARENA_MUTEX_PROF_DATA_NODE(extent_freelist)
ARENA_MUTEX_PROF_DATA_NODE(extents_dirty)
ARENA_MUTEX_PROF_DATA_NODE(extents_muzzy)
ARENA_MUTEX_PROF_DATA_NODE(extents_retained)
ARENA_MUTEX_PROF_DATA_NODE(decay_dirty)
ARENA_MUTEX_PROF_DATA_NODE(decay_muzzy)
ARENA_MUTEX_PROF_DATA_NODE(tcache_list)
static const ctl_named_node_t stats_arenas_i_mutexes_node[] = { static const ctl_named_node_t stats_arenas_i_mutexes_node[] = {
{NAME("large"), #define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)},
CHILD(named, stats_arenas_i_mutexes_large)}, ARENA_PROF_MUTEXES
{NAME("extent_freelist"), #undef OP
CHILD(named, stats_arenas_i_mutexes_extent_freelist)},
{NAME("extents_dirty"),
CHILD(named, stats_arenas_i_mutexes_extents_dirty)},
{NAME("extents_muzzy"),
CHILD(named, stats_arenas_i_mutexes_extents_muzzy)},
{NAME("extents_retained"),
CHILD(named, stats_arenas_i_mutexes_extents_retained)},
{NAME("decay_dirty"),
CHILD(named, stats_arenas_i_mutexes_decay_dirty)},
{NAME("decay_muzzy"),
CHILD(named, stats_arenas_i_mutexes_decay_muzzy)},
{NAME("tcache_list"),
CHILD(named, stats_arenas_i_mutexes_tcache_list)}
}; };
static const ctl_named_node_t stats_arenas_i_node[] = { static const ctl_named_node_t stats_arenas_i_node[] = {
@ -525,15 +470,17 @@ static const ctl_indexed_node_t stats_arenas_node[] = {
{INDEX(stats_arenas_i)} {INDEX(stats_arenas_i)}
}; };
MUTEX_PROF_DATA_NODE(mutexes_base) #define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx)
MUTEX_PROF_DATA_NODE(mutexes_prof) GLOBAL_PROF_MUTEXES
MUTEX_PROF_DATA_NODE(mutexes_ctl) #undef OP
static const ctl_named_node_t stats_mutexes_node[] = { static const ctl_named_node_t stats_mutexes_node[] = {
{NAME("base"), CHILD(named, stats_mutexes_base)}, #define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)},
{NAME("prof"), CHILD(named, stats_mutexes_prof)}, GLOBAL_PROF_MUTEXES
{NAME("ctl"), CHILD(named, stats_mutexes_ctl)}, #undef OP
{NAME("reset"), CTL(stats_mutexes_reset)} {NAME("reset"), CTL(stats_mutexes_reset)}
}; };
#undef MUTEX_PROF_DATA_NODE
static const ctl_named_node_t stats_node[] = { static const ctl_named_node_t stats_node[] = {
{NAME("allocated"), CTL(stats_allocated)}, {NAME("allocated"), CTL(stats_allocated)},
@ -545,7 +492,6 @@ static const ctl_named_node_t stats_node[] = {
{NAME("mutexes"), CHILD(named, stats_mutexes)}, {NAME("mutexes"), CHILD(named, stats_mutexes)},
{NAME("arenas"), CHILD(indexed, stats_arenas)} {NAME("arenas"), CHILD(indexed, stats_arenas)}
}; };
#undef MUTEX_PROF_DATA_NODE
static const ctl_named_node_t root_node[] = { static const ctl_named_node_t root_node[] = {
{NAME("version"), CTL(version)}, {NAME("version"), CTL(version)},
@ -784,27 +730,13 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged,
&astats->astats.decay_muzzy.purged); &astats->astats.decay_muzzy.purged);
malloc_mutex_prof_merge(&(sdstats->astats.large_mtx_data), #define OP(mtx) malloc_mutex_prof_merge( \
&(astats->astats.large_mtx_data)); &(sdstats->astats.mutex_prof_data[ \
malloc_mutex_prof_merge( arena_prof_mutex_##mtx]), \
&(sdstats->astats.extent_freelist_mtx_data), &(astats->astats.mutex_prof_data[ \
&(astats->astats.extent_freelist_mtx_data)); arena_prof_mutex_##mtx]));
malloc_mutex_prof_merge( ARENA_PROF_MUTEXES
&(sdstats->astats.extents_dirty_mtx_data), #undef OP
&(astats->astats.extents_dirty_mtx_data));
malloc_mutex_prof_merge(
&(sdstats->astats.extents_muzzy_mtx_data),
&(astats->astats.extents_muzzy_mtx_data));
malloc_mutex_prof_merge(
&(sdstats->astats.extents_retained_mtx_data),
&(astats->astats.extents_retained_mtx_data));
malloc_mutex_prof_merge(&(sdstats->astats.decay_dirty_mtx_data),
&(astats->astats.decay_dirty_mtx_data));
malloc_mutex_prof_merge(&(sdstats->astats.decay_muzzy_mtx_data),
&(astats->astats.decay_muzzy_mtx_data));
malloc_mutex_prof_merge(&(sdstats->astats.tcache_list_mtx_data),
&(astats->astats.tcache_list_mtx_data));
if (!destroyed) { if (!destroyed) {
accum_atomic_zu(&sdstats->astats.base, accum_atomic_zu(&sdstats->astats.base,
&astats->astats.base); &astats->astats.base);
@ -975,17 +907,21 @@ ctl_refresh(tsdn_t *tsdn) {
ctl_stats->retained = atomic_load_zu( ctl_stats->retained = atomic_load_zu(
&ctl_sarena->astats->astats.retained, ATOMIC_RELAXED); &ctl_sarena->astats->astats.retained, ATOMIC_RELAXED);
#define READ_GLOBAL_MUTEX_PROF_DATA(mtx, data) \ #define READ_GLOBAL_MUTEX_PROF_DATA(i, mtx) \
malloc_mutex_lock(tsdn, &mtx); \ malloc_mutex_lock(tsdn, &mtx); \
malloc_mutex_prof_read(tsdn, &ctl_stats->data, &mtx); \ malloc_mutex_prof_read(tsdn, &ctl_stats->mutex_prof_data[i], &mtx); \
malloc_mutex_unlock(tsdn, &mtx); malloc_mutex_unlock(tsdn, &mtx);
READ_GLOBAL_MUTEX_PROF_DATA(b0get()->mtx, base_mtx_data); READ_GLOBAL_MUTEX_PROF_DATA(global_prof_mutex_base,
b0get()->mtx);
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
READ_GLOBAL_MUTEX_PROF_DATA(bt2gctx_mtx, prof_mtx_data); READ_GLOBAL_MUTEX_PROF_DATA(global_prof_mutex_prof,
bt2gctx_mtx);
} }
/* We own ctl mutex already. */ /* We own ctl mutex already. */
malloc_mutex_prof_read(tsdn, &ctl_stats->ctl_mtx_data, &ctl_mtx); malloc_mutex_prof_read(tsdn,
&ctl_stats->mutex_prof_data[global_prof_mutex_ctl],
&ctl_mtx);
#undef READ_GLOBAL_MUTEX_PROF_DATA #undef READ_GLOBAL_MUTEX_PROF_DATA
} }
ctl_arenas->epoch++; ctl_arenas->epoch++;
@ -2489,29 +2425,24 @@ CTL_RO_CGEN(config_stats, stats_##n##_total_wait_time, \
CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \ CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \
nstime_ns(&l.max_wait_time), uint64_t) \ nstime_ns(&l.max_wait_time), uint64_t) \
CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \ CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \
l.max_n_thds, uint64_t) l.max_n_thds, uint32_t)
/* Global mutexes. */ /* Global mutexes. */
#define MTX(mutex) \ #define OP(mtx) \
RO_MUTEX_CTL_GEN(mutexes_##mutex, ctl_stats->mutex##_mtx_data) RO_MUTEX_CTL_GEN(mutexes_##mtx, \
ctl_stats->mutex_prof_data[global_prof_mutex_##mtx])
GLOBAL_PROF_MUTEXES GLOBAL_PROF_MUTEXES
#undef MTX #undef OP
/* Per arena mutexes */
#define OP(mtx) RO_MUTEX_CTL_GEN(arenas_i_mutexes_##mtx, \
arenas_i(mib[2])->astats->astats.mutex_prof_data[arena_prof_mutex_##mtx])
ARENA_PROF_MUTEXES
#undef OP
/* tcache bin mutex */ /* tcache bin mutex */
RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex,
arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data) arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data)
/* Per arena mutexes */
#define ARENAS_ASTATS_MUTEX_CTL_GEN(l, d) \
RO_MUTEX_CTL_GEN(arenas_i_mutexes_##l, arenas_i(mib[2])->astats->astats.d)
ARENAS_ASTATS_MUTEX_CTL_GEN(large, large_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(extent_freelist, extent_freelist_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(extents_dirty, extents_dirty_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(extents_muzzy, extents_muzzy_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(extents_retained, extents_retained_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(decay_dirty, decay_dirty_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(decay_muzzy, decay_muzzy_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(tcache_list, tcache_list_mtx_data)
#undef ARENAS_ASTATS_MUTEX_CTL_GEN
#undef RO_MUTEX_CTL_GEN #undef RO_MUTEX_CTL_GEN
/* Resets all mutex stats, including global, arena and bin mutexes. */ /* Resets all mutex stats, including global, arena and bin mutexes. */

191
src/stats.c
View File

@ -1,6 +1,18 @@
#define JEMALLOC_STATS_C_ #define JEMALLOC_STATS_C_
#include "jemalloc/internal/jemalloc_internal.h" #include "jemalloc/internal/jemalloc_internal.h"
const char *global_mutex_names[num_global_prof_mutexes] = {
#define OP(mtx) #mtx,
GLOBAL_PROF_MUTEXES
#undef OP
};
const char *arena_mutex_names[num_arena_prof_mutexes] = {
#define OP(mtx) #mtx,
ARENA_PROF_MUTEXES
#undef OP
};
#define CTL_GET(n, v, t) do { \ #define CTL_GET(n, v, t) do { \
size_t sz = sizeof(t); \ size_t sz = sizeof(t); \
xmallctl(n, (void *)v, &sz, NULL, 0); \ xmallctl(n, (void *)v, &sz, NULL, 0); \
@ -57,28 +69,49 @@ get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) {
return false; return false;
} }
#define MUTEX_CTL_STR_MAX_LENGTH 128
static void static void
gen_mutex_ctl_str(char *str, const char *prefix, const char *mutex, gen_mutex_ctl_str(char *str, size_t buf_len, const char *prefix,
const char *counter) { const char *mutex, const char *counter) {
malloc_snprintf(str, 128, "stats.%s.%s.%s", prefix, mutex, counter); malloc_snprintf(str, buf_len, "stats.%s.%s.%s", prefix, mutex, counter);
} }
static void static void
read_arena_bin_mutex_stats(unsigned arena_ind, unsigned bin_ind, read_arena_bin_mutex_stats(unsigned arena_ind, unsigned bin_ind,
uint64_t results[NUM_MUTEX_PROF_COUNTERS]) { uint64_t results[num_mutex_prof_counters]) {
char cmd[128]; char cmd[MUTEX_CTL_STR_MAX_LENGTH];
#define OP(c, t) \
gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
"arenas.0.bins.0","mutex", #c); \
CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \
(t *)&results[mutex_counter_##c], t);
MUTEX_PROF_COUNTERS
#undef OP
}
unsigned i; static void
for (i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) { mutex_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque,
gen_mutex_ctl_str(cmd, "arenas.0.bins.0","mutex", const char *name, uint64_t stats[num_mutex_prof_counters],
mutex_counter_names[i]); const char *json_indent, bool last) {
CTL_M2_M4_GET(cmd, arena_ind, bin_ind, &results[i], uint64_t); malloc_cprintf(write_cb, cbopaque, "%s\"%s\": {\n", json_indent, name);
}
mutex_prof_counter_ind_t k = 0;
char *fmt_str[2] = {"%s\t\"%s\": %"FMTu32"%s\n",
"%s\t\"%s\": %"FMTu64"%s\n"};
#define OP(c, t) \
malloc_cprintf(write_cb, cbopaque, \
fmt_str[sizeof(t) / sizeof(uint32_t) - 1], \
json_indent, #c, (t)stats[mutex_counter_##c], \
(++k == num_mutex_prof_counters) ? "" : ",");
MUTEX_PROF_COUNTERS
#undef OP
malloc_cprintf(write_cb, cbopaque, "%s}%s\n", json_indent,
last ? "" : ",");
} }
static void static void
stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, bool large, unsigned i) { bool json, bool large, bool mutex, unsigned i) {
size_t page; size_t page;
bool in_gap, in_gap_prev; bool in_gap, in_gap_prev;
unsigned nbins, j; unsigned nbins, j;
@ -147,9 +180,6 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
size_t); size_t);
if (json) { if (json) {
uint64_t mutex_stats[NUM_MUTEX_PROF_COUNTERS];
read_arena_bin_mutex_stats(i, j, mutex_stats);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\t{\n" "\t\t\t\t\t{\n"
"\t\t\t\t\t\t\"nmalloc\": %"FMTu64",\n" "\t\t\t\t\t\t\"nmalloc\": %"FMTu64",\n"
@ -169,20 +199,16 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
} }
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\t\t\"nreslabs\": %"FMTu64",\n" "\t\t\t\t\t\t\"nreslabs\": %"FMTu64",\n"
"\t\t\t\t\t\t\"curslabs\": %zu,\n" "\t\t\t\t\t\t\"curslabs\": %zu%s\n",
"\t\t\t\t\t\t\"mutex\": {\n", nreslabs, curslabs, mutex ? "," : "");
nreslabs,
curslabs);
for (unsigned k = 0; k < NUM_MUTEX_PROF_COUNTERS; k++) { if (mutex) {
malloc_cprintf(write_cb, cbopaque, uint64_t mutex_stats[num_mutex_prof_counters];
"\t\t\t\t\t\t\t\"%s\": %"FMTu64"%s\n", read_arena_bin_mutex_stats(i, j, mutex_stats);
mutex_counter_names[k], mutex_stats[k], mutex_stats_output_json(write_cb, cbopaque,
k == NUM_MUTEX_PROF_COUNTERS - 1 ? "" : ","); "mutex", mutex_stats, "\t\t\t\t\t\t", true);
} }
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\t\t}\n"
"\t\t\t\t\t}%s\n", "\t\t\t\t\t}%s\n",
(j + 1 < nbins) ? "," : ""); (j + 1 < nbins) ? "," : "");
} else if (!in_gap) { } else if (!in_gap) {
@ -326,74 +352,79 @@ stats_arena_lextents_print(void (*write_cb)(void *, const char *),
static void static void
read_arena_mutex_stats(unsigned arena_ind, read_arena_mutex_stats(unsigned arena_ind,
uint64_t results[NUM_ARENA_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]) { uint64_t results[num_arena_prof_mutexes][num_mutex_prof_counters]) {
char cmd[128]; char cmd[MUTEX_CTL_STR_MAX_LENGTH];
unsigned i, j; arena_prof_mutex_ind_t i;
for (i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { for (i = 0; i < num_arena_prof_mutexes; i++) {
for (j = 0; j < NUM_MUTEX_PROF_COUNTERS; j++) { #define OP(c, t) \
gen_mutex_ctl_str(cmd, "arenas.0.mutexes", gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
arena_mutex_names[i], mutex_counter_names[j]); "arenas.0.mutexes", arena_mutex_names[i], #c); \
CTL_M2_GET(cmd, arena_ind, &results[i][j], uint64_t); CTL_M2_GET(cmd, arena_ind, \
} (t *)&results[i][mutex_counter_##c], t);
MUTEX_PROF_COUNTERS
#undef OP
} }
} }
static void
mutex_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque,
const char *name, uint64_t stats[NUM_MUTEX_PROF_COUNTERS],
const char *json_indent, bool last) {
malloc_cprintf(write_cb, cbopaque, "%s\"%s\": {\n", json_indent, name);
for (unsigned i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) {
malloc_cprintf(write_cb, cbopaque, "%s\t\"%s\": %"FMTu64"%s\n",
json_indent, mutex_counter_names[i], stats[i],
i < (NUM_MUTEX_PROF_COUNTERS - 1) ? "," : "");
}
malloc_cprintf(write_cb, cbopaque, "%s}%s\n", json_indent,
last ? "" : ",");
}
static void static void
mutex_stats_output(void (*write_cb)(void *, const char *), void *cbopaque, mutex_stats_output(void (*write_cb)(void *, const char *), void *cbopaque,
const char *name, uint64_t stats[NUM_MUTEX_PROF_COUNTERS], const char *name, uint64_t stats[num_mutex_prof_counters],
bool first_mutex) { bool first_mutex) {
if (first_mutex) { if (first_mutex) {
/* Print title. */ /* Print title. */
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
" n_lock_ops n_waiting" " n_lock_ops n_waiting"
" n_spin_acq n_owner_switch total_wait_ns" " n_spin_acq n_owner_switch total_wait_ns"
" max_wait_ns max_n_wait_thds\n"); " max_wait_ns max_n_thds\n");
} }
malloc_cprintf(write_cb, cbopaque, "%s", name); malloc_cprintf(write_cb, cbopaque, "%s", name);
malloc_cprintf(write_cb, cbopaque, ":%*c", malloc_cprintf(write_cb, cbopaque, ":%*c",
(int)(19 - strlen(name)), ' '); (int)(20 - strlen(name)), ' ');
for (unsigned i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) { char *fmt_str[2] = {"%12"FMTu32, "%16"FMTu64};
malloc_cprintf(write_cb, cbopaque, " %16"FMTu64, stats[i]); #define OP(c, t) \
} malloc_cprintf(write_cb, cbopaque, \
fmt_str[sizeof(t) / sizeof(uint32_t) - 1], \
(t)stats[mutex_counter_##c]);
MUTEX_PROF_COUNTERS
#undef OP
malloc_cprintf(write_cb, cbopaque, "\n"); malloc_cprintf(write_cb, cbopaque, "\n");
} }
static void static void
stats_arena_mutexes_print(void (*write_cb)(void *, const char *), stats_arena_mutexes_print(void (*write_cb)(void *, const char *),
void *cbopaque, bool json, bool json_end, unsigned arena_ind) { void *cbopaque, bool json, bool json_end, unsigned arena_ind) {
uint64_t mutex_stats[NUM_ARENA_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]; uint64_t mutex_stats[num_arena_prof_mutexes][num_mutex_prof_counters];
read_arena_mutex_stats(arena_ind, mutex_stats); read_arena_mutex_stats(arena_ind, mutex_stats);
/* Output mutex stats. */ /* Output mutex stats. */
if (json) { if (json) {
malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"mutexes\": {\n"); malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"mutexes\": {\n");
for (unsigned i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { arena_prof_mutex_ind_t i, last_mutex;
last_mutex = num_arena_prof_mutexes - 1;
if (!config_tcache) {
last_mutex--;
}
for (i = 0; i < num_arena_prof_mutexes; i++) {
if (!config_tcache &&
i == arena_prof_mutex_tcache_list) {
continue;
}
mutex_stats_output_json(write_cb, cbopaque, mutex_stats_output_json(write_cb, cbopaque,
arena_mutex_names[i], mutex_stats[i], arena_mutex_names[i], mutex_stats[i],
"\t\t\t\t\t", (i == NUM_ARENA_PROF_MUTEXES - 1)); "\t\t\t\t\t", (i == last_mutex));
} }
malloc_cprintf(write_cb, cbopaque, "\t\t\t\t}%s\n", malloc_cprintf(write_cb, cbopaque, "\t\t\t\t}%s\n",
json_end ? "" : ","); json_end ? "" : ",");
} else { } else {
for (unsigned i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) { arena_prof_mutex_ind_t i;
for (i = 0; i < num_arena_prof_mutexes; i++) {
if (!config_tcache &&
i == arena_prof_mutex_tcache_list) {
continue;
}
mutex_stats_output(write_cb, cbopaque, mutex_stats_output(write_cb, cbopaque,
arena_mutex_names[i], mutex_stats[i], i == 0); arena_mutex_names[i], mutex_stats[i], i == 0);
} }
@ -636,7 +667,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
!(bins || large), i); !(bins || large), i);
} }
if (bins) { if (bins) {
stats_arena_bins_print(write_cb, cbopaque, json, large, i); stats_arena_bins_print(write_cb, cbopaque, json, large, mutex,
i);
} }
if (large) { if (large) {
stats_arena_lextents_print(write_cb, cbopaque, json, i); stats_arena_lextents_print(write_cb, cbopaque, json, i);
@ -995,16 +1027,17 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
static void static void
read_global_mutex_stats( read_global_mutex_stats(
uint64_t results[NUM_GLOBAL_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]) { uint64_t results[num_global_prof_mutexes][num_mutex_prof_counters]) {
char cmd[128]; char cmd[MUTEX_CTL_STR_MAX_LENGTH];
unsigned i, j; global_prof_mutex_ind_t i;
for (i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { for (i = 0; i < num_global_prof_mutexes; i++) {
for (j = 0; j < NUM_MUTEX_PROF_COUNTERS; j++) { #define OP(c, t) \
gen_mutex_ctl_str(cmd, "mutexes", global_mutex_names[i], gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \
mutex_counter_names[j]); "mutexes", global_mutex_names[i], #c); \
CTL_GET(cmd, &results[i][j], uint64_t); CTL_GET(cmd, (t *)&results[i][mutex_counter_##c], t);
} MUTEX_PROF_COUNTERS
#undef OP
} }
} }
@ -1021,7 +1054,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_GET("stats.mapped", &mapped, size_t); CTL_GET("stats.mapped", &mapped, size_t);
CTL_GET("stats.retained", &retained, size_t); CTL_GET("stats.retained", &retained, size_t);
uint64_t mutex_stats[NUM_GLOBAL_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]; uint64_t mutex_stats[num_global_prof_mutexes][num_mutex_prof_counters];
if (mutex) { if (mutex) {
read_global_mutex_stats(mutex_stats); read_global_mutex_stats(mutex_stats);
} }
@ -1041,15 +1074,16 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"mapped\": %zu,\n", mapped); "\t\t\t\"mapped\": %zu,\n", mapped);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"retained\": %zu,\n", retained); "\t\t\t\"retained\": %zu%s\n", retained, mutex ? "," : "");
if (mutex) { if (mutex) {
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"mutexes\": {\n"); "\t\t\t\"mutexes\": {\n");
global_prof_mutex_ind_t i;
for (unsigned i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { for (i = 0; i < num_global_prof_mutexes; i++) {
mutex_stats_output_json(write_cb, cbopaque, mutex_stats_output_json(write_cb, cbopaque,
global_mutex_names[i], mutex_stats[i], global_mutex_names[i], mutex_stats[i],
"\t\t\t\t", i == NUM_GLOBAL_PROF_MUTEXES - 1); "\t\t\t\t",
i == num_global_prof_mutexes - 1);
} }
malloc_cprintf(write_cb, cbopaque, "\t\t\t}\n"); malloc_cprintf(write_cb, cbopaque, "\t\t\t}\n");
} }
@ -1061,7 +1095,8 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
" resident: %zu, mapped: %zu, retained: %zu\n", " resident: %zu, mapped: %zu, retained: %zu\n",
allocated, active, metadata, resident, mapped, retained); allocated, active, metadata, resident, mapped, retained);
if (mutex) { if (mutex) {
for (unsigned i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) { global_prof_mutex_ind_t i;
for (i = 0; i < num_global_prof_mutexes; i++) {
mutex_stats_output(write_cb, cbopaque, mutex_stats_output(write_cb, cbopaque,
global_mutex_names[i], mutex_stats[i], global_mutex_names[i], mutex_stats[i],
i == 0); i == 0);