project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added "stats.mutexes.reset" mallctl to reset all mutex stats.

Browse Source 
Also switched from the term "lock" to "mutex".
This commit is contained in:
Qi Wang 2017-03-13 17:29:03 -07:00 committed by Qi Wang
parent bd2006a41b
commit 64c5f5c174
12 changed files with 248 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/jemalloc/internal/ctl_externs.h
View File

@ -4,13 +4,13 @@
/* Maximum ctl tree depth. */
#define CTL_MAX_DEPTH 7
#define NUM_GLOBAL_PROF_LOCKS 3
#define NUM_ARENA_PROF_LOCKS 6
#define NUM_LOCK_PROF_COUNTERS 7
#define NUM_GLOBAL_PROF_MUTEXES 3
#define NUM_ARENA_PROF_MUTEXES 6
#define NUM_MUTEX_PROF_COUNTERS 7
extern const char *arena_lock_names[NUM_ARENA_PROF_LOCKS];
extern const char *global_lock_names[NUM_GLOBAL_PROF_LOCKS];
extern const char *lock_counter_names[NUM_LOCK_PROF_COUNTERS];
extern const char *arena_mutex_names[NUM_ARENA_PROF_MUTEXES];
extern const char *global_mutex_names[NUM_GLOBAL_PROF_MUTEXES];
extern const char *mutex_counter_names[NUM_MUTEX_PROF_COUNTERS];
int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
void *newp, size_t newlen);

4
include/jemalloc/internal/ctl_structs.h
View File

@ -42,8 +42,8 @@ struct ctl_stats_s {
size_t mapped;
size_t retained;
#define MTX(mutex) lock_prof_data_t mutex##_mtx_data;
GLOBAL_PROF_MUTEXES
#define MTX(mutex) mutex_prof_data_t mutex##_mtx_data;
GLOBAL_PROF_MUTEXES
#undef MTX
};

1
include/jemalloc/internal/mutex_externs.h
View File

@ -14,5 +14,6 @@ void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
bool malloc_mutex_boot(void);
void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);
#endif /* JEMALLOC_INTERNAL_MUTEX_EXTERNS_H */

14
include/jemalloc/internal/mutex_inlines.h
View File

@ -9,9 +9,9 @@ bool malloc_mutex_trylock(malloc_mutex_t *mutex);
void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_lock_prof_read(tsdn_t *tsdn, lock_prof_data_t *data,
void malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
malloc_mutex_t *mutex);
void malloc_lock_prof_merge(lock_prof_data_t *sum, lock_prof_data_t *data);
void malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
@ -28,7 +28,7 @@ malloc_mutex_trylock(malloc_mutex_t *mutex) {
/* Aggregate lock prof data. */
JEMALLOC_INLINE void
malloc_lock_prof_merge(lock_prof_data_t *sum, lock_prof_data_t *data) {
malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
sum->tot_wait_time += data->tot_wait_time;
if (data->max_wait_time > sum->max_wait_time) {
sum->max_wait_time = data->max_wait_time;
@ -52,7 +52,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
malloc_mutex_lock_slow(mutex);
}
/* We own the lock now. Update a few counters. */
lock_prof_data_t *data = &mutex->prof_data;
mutex_prof_data_t *data = &mutex->prof_data;
data->n_lock_ops++;
if (data->prev_owner != tsdn) {
data->prev_owner = tsdn;
@ -82,10 +82,10 @@ malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
/* Copy the prof data from mutex for processing. */
JEMALLOC_INLINE void
malloc_lock_prof_read(tsdn_t *tsdn, lock_prof_data_t *data,
malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
malloc_mutex_t *mutex) {
lock_prof_data_t *source = &mutex->prof_data;
/* Can only read with the lock. */
mutex_prof_data_t *source = &mutex->prof_data;
/* Can only read holding the mutex. */
malloc_mutex_assert_owner(tsdn, mutex);
*data = *source;

20
include/jemalloc/internal/mutex_structs.h
View File

@ -1,20 +1,20 @@
#ifndef JEMALLOC_INTERNAL_MUTEX_STRUCTS_H
#define JEMALLOC_INTERNAL_MUTEX_STRUCTS_H
struct lock_prof_data_s {
struct mutex_prof_data_s {
/*
* Counters touched on the slow path, i.e. when there is lock
* contention. We update them once we have the lock.
*/
/* Total time (in nano seconds) spent waiting on this lock. */
/* Total time (in nano seconds) spent waiting on this mutex. */
uint64_t tot_wait_time;
/* Max time (in nano seconds) spent on a single lock operation. */
uint64_t max_wait_time;
/* # of times have to wait for this lock (after spinning). */
/* # of times have to wait for this mutex (after spinning). */
uint64_t n_wait_times;
/* # of times acquired the lock through local spinning. */
/* # of times acquired the mutex through local spinning. */
uint64_t n_spin_acquired;
/* Max # of threads waiting for the lock at the same time. */
/* Max # of threads waiting for the mutex at the same time. */
uint32_t max_n_thds;
/* Current # of threads waiting on the lock. Atomic synced. */
uint32_t n_waiting_thds;
@ -25,9 +25,9 @@ struct lock_prof_data_s {
* the lock) so that we have a higher chance of them being on the same
* cacheline.
*/
/* # of times the new lock holder is different from the previous one. */
/* # of times the mutex holder is different than the previous one. */
uint64_t n_owner_switches;
/* Previous lock holder, to facilitate n_owner_switches. */
/* Previous mutex holder, to facilitate n_owner_switches. */
tsdn_t *prev_owner;
/* # of lock() operations in total. */
uint64_t n_lock_ops;
@ -38,13 +38,13 @@ struct malloc_mutex_s {
struct {
/*
* prof_data is defined first to reduce cacheline
* bouncing: the data is not touched by the lock holder
* bouncing: the data is not touched by the mutex holder
* during unlocking, while might be modified by
* contenders. Having it before the lock itself could
* contenders. Having it before the mutex itself could
* avoid prefetching a modified cacheline (for the
* unlocking thread).
*/
lock_prof_data_t prof_data;
mutex_prof_data_t prof_data;
#ifdef _WIN32
# if _WIN32_WINNT >= 0x0600
SRWLOCK lock;

2
include/jemalloc/internal/mutex_types.h
View File

@ -1,7 +1,7 @@
#ifndef JEMALLOC_INTERNAL_MUTEX_TYPES_H
#define JEMALLOC_INTERNAL_MUTEX_TYPES_H
typedef struct lock_prof_data_s lock_prof_data_t;
typedef struct mutex_prof_data_s mutex_prof_data_t;
typedef struct malloc_mutex_s malloc_mutex_t;
#ifdef _WIN32

1
include/jemalloc/internal/private_symbols.txt
View File

@ -269,6 +269,7 @@ lg_floor
lg_prof_sample
malloc_cprintf
malloc_getcpu
malloc_mutex_prof_data_reset
malloc_mutex_assert_not_owner
malloc_mutex_assert_owner
malloc_mutex_boot

14
include/jemalloc/internal/stats_structs.h
View File

@ -57,7 +57,7 @@ struct malloc_bin_stats_s {
/* Current number of slabs in this bin. */
size_t curslabs;
lock_prof_data_t lock_data;
mutex_prof_data_t mutex_data;
};
struct malloc_large_stats_s {
@ -124,12 +124,12 @@ struct arena_stats_s {
/* Number of bytes cached in tcache associated with this arena. */
atomic_zu_t tcache_bytes; /* Derived. */
lock_prof_data_t large_mtx_data;
lock_prof_data_t extent_freelist_mtx_data;
lock_prof_data_t extents_cached_mtx_data;
lock_prof_data_t extents_retained_mtx_data;
lock_prof_data_t decay_mtx_data;
lock_prof_data_t tcache_mtx_data;
mutex_prof_data_t large_mtx_data;
mutex_prof_data_t extent_freelist_mtx_data;
mutex_prof_data_t extents_cached_mtx_data;
mutex_prof_data_t extents_retained_mtx_data;
mutex_prof_data_t decay_mtx_data;
mutex_prof_data_t tcache_mtx_data;
/* One element for each large size class. */
malloc_large_stats_t lstats[NSIZES - NBINS];

6
src/arena.c
View File

@ -292,14 +292,14 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
tbin->ncached * index2size(i));
}
}
malloc_lock_prof_read(tsdn, &astats->tcache_mtx_data,
malloc_mutex_prof_read(tsdn, &astats->tcache_mtx_data,
&arena->tcache_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
}
#define READ_ARENA_MUTEX_PROF_DATA(mtx, data) \
malloc_mutex_lock(tsdn, &arena->mtx); \
malloc_lock_prof_read(tsdn, &astats->data, &arena->mtx); \
malloc_mutex_prof_read(tsdn, &astats->data, &arena->mtx); \
malloc_mutex_unlock(tsdn, &arena->mtx);
/* Gather per arena mutex profiling data. */
@ -317,7 +317,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_bin_t *bin = &arena->bins[i];
malloc_mutex_lock(tsdn, &bin->lock);
malloc_lock_prof_read(tsdn, &bstats[i].lock_data, &bin->lock);
malloc_mutex_prof_read(tsdn, &bstats[i].mutex_data, &bin->lock);
bstats[i].nmalloc += bin->stats.nmalloc;
bstats[i].ndalloc += bin->stats.ndalloc;
bstats[i].nrequests += bin->stats.nrequests;

200
src/ctl.c
View File

@ -13,13 +13,13 @@ static bool ctl_initialized;
static ctl_stats_t *ctl_stats;
static ctl_arenas_t *ctl_arenas;
const char *global_lock_names[NUM_GLOBAL_PROF_LOCKS] = {
const char *global_mutex_names[NUM_GLOBAL_PROF_MUTEXES] = {
"base",
"prof",
"ctl"
};
const char *arena_lock_names[NUM_ARENA_PROF_LOCKS] = {
const char *arena_mutex_names[NUM_ARENA_PROF_MUTEXES] = {
"large",
"extent_freelist",
"extents_cached",
@ -28,7 +28,7 @@ const char *arena_lock_names[NUM_ARENA_PROF_LOCKS] = {
"tcache"
};
const char *lock_counter_names[NUM_LOCK_PROF_COUNTERS] = {
const char *mutex_counter_names[NUM_MUTEX_PROF_COUNTERS] = {
"num_ops",
"num_wait",
"num_spin_acq",
@ -203,7 +203,7 @@ CTL_PROTO(stats_resident)
CTL_PROTO(stats_mapped)
CTL_PROTO(stats_retained)
#define LOCK_STATS_CTL_PROTO_GEN(n) \
#define MUTEX_STATS_CTL_PROTO_GEN(n) \
CTL_PROTO(stats_##n##_num_ops) \
CTL_PROTO(stats_##n##_num_wait) \
CTL_PROTO(stats_##n##_num_spin_acq) \
@ -212,25 +212,27 @@ CTL_PROTO(stats_##n##_total_wait_time) \
CTL_PROTO(stats_##n##_max_wait_time) \
CTL_PROTO(stats_##n##_max_num_thds)
/* Global locks. */
LOCK_STATS_CTL_PROTO_GEN(locks_base)
LOCK_STATS_CTL_PROTO_GEN(locks_prof)
LOCK_STATS_CTL_PROTO_GEN(locks_ctl)
/* Global mutexes. */
MUTEX_STATS_CTL_PROTO_GEN(mutexes_base)
MUTEX_STATS_CTL_PROTO_GEN(mutexes_prof)
MUTEX_STATS_CTL_PROTO_GEN(mutexes_ctl)
/* Arena bin locks. */
LOCK_STATS_CTL_PROTO_GEN(arenas_i_bins_j_lock)
/* Arena bin mutexes. */
MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex)
#define ARENA_LOCKS_CTL_PROTO_GEN(n) \
LOCK_STATS_CTL_PROTO_GEN(arenas_i_locks_##n)
/* Per arena locks. */
ARENA_LOCKS_CTL_PROTO_GEN(large)
ARENA_LOCKS_CTL_PROTO_GEN(extent_freelist)
ARENA_LOCKS_CTL_PROTO_GEN(extents_cached)
ARENA_LOCKS_CTL_PROTO_GEN(extents_retained)
ARENA_LOCKS_CTL_PROTO_GEN(decay)
ARENA_LOCKS_CTL_PROTO_GEN(tcache)
#undef ARENA_LOCKS_CTL_PROTO_GEN
#undef LOCK_STATS_CTL_PROTO_GEN
#define ARENA_MUTEXES_CTL_PROTO_GEN(n) \
MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##n)
/* Per arena mutexes. */
ARENA_MUTEXES_CTL_PROTO_GEN(large)
ARENA_MUTEXES_CTL_PROTO_GEN(extent_freelist)
ARENA_MUTEXES_CTL_PROTO_GEN(extents_cached)
ARENA_MUTEXES_CTL_PROTO_GEN(extents_retained)
ARENA_MUTEXES_CTL_PROTO_GEN(decay)
ARENA_MUTEXES_CTL_PROTO_GEN(tcache)
#undef ARENA_MUTEXES_CTL_PROTO_GEN
#undef MUTEX_STATS_CTL_PROTO_GEN
CTL_PROTO(stats_mutexes_reset)
/******************************************************************************/
/* mallctl tree. */
@ -399,7 +401,7 @@ static const ctl_named_node_t stats_arenas_i_large_node[] = {
{NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}
};
#define LOCK_PROF_DATA_NODE(prefix) \
#define MUTEX_PROF_DATA_NODE(prefix) \
static const ctl_named_node_t stats_##prefix##_node[] = { \
{NAME("num_ops"), \
CTL(stats_##prefix##_num_ops)}, \
@ -418,7 +420,7 @@ static const ctl_named_node_t stats_##prefix##_node[] = { \
/* Note that # of current waiting thread not provided. */ \
};
LOCK_PROF_DATA_NODE(arenas_i_bins_j_lock)
MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex)
static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
{NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)},
@ -430,7 +432,7 @@ static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
{NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)},
{NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)},
{NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)},
{NAME("lock"), CHILD(named, stats_arenas_i_bins_j_lock)}
{NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)}
};
static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = {
@ -455,25 +457,25 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
{INDEX(stats_arenas_i_lextents_j)}
};
#define ARENA_LOCK_PROF_DATA_NODE(n) LOCK_PROF_DATA_NODE(arenas_i_locks_##n)
#define ARENA_MUTEX_PROF_DATA_NODE(n) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##n)
ARENA_LOCK_PROF_DATA_NODE(large)
ARENA_LOCK_PROF_DATA_NODE(extent_freelist)
ARENA_LOCK_PROF_DATA_NODE(extents_cached)
ARENA_LOCK_PROF_DATA_NODE(extents_retained)
ARENA_LOCK_PROF_DATA_NODE(decay)
ARENA_LOCK_PROF_DATA_NODE(tcache)
ARENA_MUTEX_PROF_DATA_NODE(large)
ARENA_MUTEX_PROF_DATA_NODE(extent_freelist)
ARENA_MUTEX_PROF_DATA_NODE(extents_cached)
ARENA_MUTEX_PROF_DATA_NODE(extents_retained)
ARENA_MUTEX_PROF_DATA_NODE(decay)
ARENA_MUTEX_PROF_DATA_NODE(tcache)
static const ctl_named_node_t stats_arenas_i_locks_node[] = {
{NAME("large"), CHILD(named, stats_arenas_i_locks_large)},
static const ctl_named_node_t stats_arenas_i_mutexes_node[] = {
{NAME("large"), CHILD(named, stats_arenas_i_mutexes_large)},
{NAME("extent_freelist"),
CHILD(named, stats_arenas_i_locks_extent_freelist)},
CHILD(named, stats_arenas_i_mutexes_extent_freelist)},
{NAME("extents_cached"),
CHILD(named, stats_arenas_i_locks_extents_cached)},
CHILD(named, stats_arenas_i_mutexes_extents_cached)},
{NAME("extents_retained"),
CHILD(named, stats_arenas_i_locks_extents_retained)},
{NAME("decay"), CHILD(named, stats_arenas_i_locks_decay)},
{NAME("tcache"), CHILD(named, stats_arenas_i_locks_tcache)}
CHILD(named, stats_arenas_i_mutexes_extents_retained)},
{NAME("decay"), CHILD(named, stats_arenas_i_mutexes_decay)},
{NAME("tcache"), CHILD(named, stats_arenas_i_mutexes_tcache)}
};
static const ctl_named_node_t stats_arenas_i_node[] = {
@ -500,7 +502,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
{NAME("large"), CHILD(named, stats_arenas_i_large)},
{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
{NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)},
{NAME("locks"), CHILD(named, stats_arenas_i_locks)}
{NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)}
};
static const ctl_named_node_t super_stats_arenas_i_node[] = {
{NAME(""), CHILD(named, stats_arenas_i)}
@ -510,13 +512,14 @@ static const ctl_indexed_node_t stats_arenas_node[] = {
{INDEX(stats_arenas_i)}
};
LOCK_PROF_DATA_NODE(locks_base)
LOCK_PROF_DATA_NODE(locks_prof)
LOCK_PROF_DATA_NODE(locks_ctl)
static const ctl_named_node_t stats_locks_node[] = {
{NAME("base"), CHILD(named, stats_locks_base)},
{NAME("prof"), CHILD(named, stats_locks_prof)},
{NAME("ctl"), CHILD(named, stats_locks_ctl)}
MUTEX_PROF_DATA_NODE(mutexes_base)
MUTEX_PROF_DATA_NODE(mutexes_prof)
MUTEX_PROF_DATA_NODE(mutexes_ctl)
static const ctl_named_node_t stats_mutexes_node[] = {
{NAME("base"), CHILD(named, stats_mutexes_base)},
{NAME("prof"), CHILD(named, stats_mutexes_prof)},
{NAME("ctl"), CHILD(named, stats_mutexes_ctl)},
{NAME("reset"), CTL(stats_mutexes_reset)}
};
static const ctl_named_node_t stats_node[] = {
@ -526,10 +529,10 @@ static const ctl_named_node_t stats_node[] = {
{NAME("resident"), CTL(stats_resident)},
{NAME("mapped"), CTL(stats_mapped)},
{NAME("retained"), CTL(stats_retained)},
{NAME("locks"), CHILD(named, stats_locks)},
{NAME("mutexes"), CHILD(named, stats_mutexes)},
{NAME("arenas"), CHILD(indexed, stats_arenas)}
};
#undef LOCK_PROF_DATA_NODE
#undef MUTEX_PROF_DATA_NODE
static const ctl_named_node_t root_node[] = {
{NAME("version"), CTL(version)},
@ -768,20 +771,20 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged,
&astats->astats.decay_muzzy.purged);
malloc_lock_prof_merge(&(sdstats->astats.large_mtx_data),
malloc_mutex_prof_merge(&(sdstats->astats.large_mtx_data),
&(astats->astats.large_mtx_data));
malloc_lock_prof_merge(
malloc_mutex_prof_merge(
&(sdstats->astats.extent_freelist_mtx_data),
&(astats->astats.extent_freelist_mtx_data));
malloc_lock_prof_merge(
malloc_mutex_prof_merge(
&(sdstats->astats.extents_cached_mtx_data),
&(astats->astats.extents_cached_mtx_data));
malloc_lock_prof_merge(
malloc_mutex_prof_merge(
&(sdstats->astats.extents_retained_mtx_data),
&(astats->astats.extents_retained_mtx_data));
malloc_lock_prof_merge(&(sdstats->astats.decay_mtx_data),
malloc_mutex_prof_merge(&(sdstats->astats.decay_mtx_data),
&(astats->astats.decay_mtx_data));
malloc_lock_prof_merge(&(sdstats->astats.tcache_mtx_data),
malloc_mutex_prof_merge(&(sdstats->astats.tcache_mtx_data),
&(astats->astats.tcache_mtx_data));
if (!destroyed) {
@ -849,8 +852,8 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
} else {
assert(astats->bstats[i].curslabs == 0);
}
malloc_lock_prof_merge(&sdstats->bstats[i].lock_data,
&astats->bstats[i].lock_data);
malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data,
&astats->bstats[i].mutex_data);
}
for (i = 0; i < NSIZES - NBINS; i++) {
@ -956,7 +959,7 @@ ctl_refresh(tsdn_t *tsdn) {
#define READ_GLOBAL_MUTEX_PROF_DATA(mtx, data) \
malloc_mutex_lock(tsdn, &mtx); \
malloc_lock_prof_read(tsdn, &ctl_stats->data, &mtx); \
malloc_mutex_prof_read(tsdn, &ctl_stats->data, &mtx); \
malloc_mutex_unlock(tsdn, &mtx);
READ_GLOBAL_MUTEX_PROF_DATA(b0get()->mtx, base_mtx_data);
@ -964,7 +967,7 @@ ctl_refresh(tsdn_t *tsdn) {
READ_GLOBAL_MUTEX_PROF_DATA(bt2gctx_mtx, prof_mtx_data);
}
/* We own ctl mutex already. */
malloc_lock_prof_read(tsdn, &ctl_stats->ctl_mtx_data, &ctl_mtx);
malloc_mutex_prof_read(tsdn, &ctl_stats->ctl_mtx_data, &ctl_mtx);
#undef READ_GLOBAL_MUTEX_PROF_DATA
}
ctl_arenas->epoch++;
@ -2454,7 +2457,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests,
uint64_t) /* Intentional. */
/* Lock profiling related APIs below. */
#define RO_LOCK_CTL_GEN(n, l) \
#define RO_MUTEX_CTL_GEN(n, l) \
CTL_RO_CGEN(config_stats, stats_##n##_num_ops, \
l.n_lock_ops, uint64_t) \
CTL_RO_CGEN(config_stats, stats_##n##_num_wait, \
@ -2470,32 +2473,81 @@ CTL_RO_CGEN(config_stats, stats_##n##_max_wait_time, \
CTL_RO_CGEN(config_stats, stats_##n##_max_num_thds, \
l.max_n_thds, uint64_t)
/* Global lock. */
#define MTX(mutex) RO_LOCK_CTL_GEN(locks_##mutex, ctl_stats->mutex##_mtx_data)
/* Global mutexes. */
#define MTX(mutex) \
RO_MUTEX_CTL_GEN(mutexes_##mutex, ctl_stats->mutex##_mtx_data)
GLOBAL_PROF_MUTEXES
#undef MTX
/* arena->bins[j].lock */
RO_LOCK_CTL_GEN(arenas_i_bins_j_lock,
arenas_i(mib[2])->astats->bstats[mib[4]].lock_data)
RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex,
arenas_i(mib[2])->astats->bstats[mib[4]].mutex_data)
/* Per arena locks */
#define ARENAS_ASTATS_LOCK_CTL_GEN(l, d) \
RO_LOCK_CTL_GEN(arenas_i_locks_##l, arenas_i(mib[2])->astats->astats.d)
/* Per arena mutexes */
#define ARENAS_ASTATS_MUTEX_CTL_GEN(l, d) \
RO_MUTEX_CTL_GEN(arenas_i_mutexes_##l, arenas_i(mib[2])->astats->astats.d)
/* arena->large_mtx */
ARENAS_ASTATS_LOCK_CTL_GEN(large, large_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(large, large_mtx_data)
/* arena->extent_freelist_mtx */
ARENAS_ASTATS_LOCK_CTL_GEN(extent_freelist, extent_freelist_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(extent_freelist, extent_freelist_mtx_data)
/* arena->extents_cached.mtx */
ARENAS_ASTATS_LOCK_CTL_GEN(extents_cached, extents_cached_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(extents_cached, extents_cached_mtx_data)
/* arena->extents_retained.mtx */
ARENAS_ASTATS_LOCK_CTL_GEN(extents_retained, extents_retained_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(extents_retained, extents_retained_mtx_data)
/* arena->decay.mtx */
ARENAS_ASTATS_LOCK_CTL_GEN(decay, decay_mtx_data)
ARENAS_ASTATS_MUTEX_CTL_GEN(decay, decay_mtx_data)
/* arena->tcache_ql_mtx */
ARENAS_ASTATS_LOCK_CTL_GEN(tcache, tcache_mtx_data)
#undef ARENAS_ASTATS_LOCK_CTL_GEN
#undef RO_LOCK_CTL_GEN
ARENAS_ASTATS_MUTEX_CTL_GEN(tcache, tcache_mtx_data)
#undef ARENAS_ASTATS_MUTEX_CTL_GEN
#undef RO_MUTEX_CTL_GEN
/* Resets all mutex stats, including global, arena and bin mutexes. */
static int
stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
if (!config_stats) {
return ENOENT;
}
tsdn_t *tsdn = tsd_tsdn(tsd);
#define MUTEX_PROF_RESET(mtx) \
malloc_mutex_lock(tsdn, &mtx); \
malloc_mutex_prof_data_reset(tsdn, &mtx); \
malloc_mutex_unlock(tsdn, &mtx);
/* Global mutexes: base, prof and ctl. */
MUTEX_PROF_RESET(b0get()->mtx);
if (config_prof && opt_prof) {
MUTEX_PROF_RESET(bt2gctx_mtx);
}
MUTEX_PROF_RESET(ctl_mtx);
/* Per arena mutexes. */
unsigned n = narenas_total_get();
for (unsigned i = 0; i < n; i++) {
arena_t *arena = arena_get(tsdn, i, false);
if (!arena) {
continue;
}
MUTEX_PROF_RESET(arena->large_mtx);
MUTEX_PROF_RESET(arena->extent_freelist_mtx);
MUTEX_PROF_RESET(arena->extents_cached.mtx);
MUTEX_PROF_RESET(arena->extents_retained.mtx);
MUTEX_PROF_RESET(arena->decay.mtx);
if (config_tcache) {
MUTEX_PROF_RESET(arena->tcache_ql_mtx);
}
for (szind_t i = 0; i < NBINS; i++) {
arena_bin_t *bin = &arena->bins[i];
MUTEX_PROF_RESET(bin->lock);
}
}
#undef MUTEX_PROF_RESET
return 0;
}
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc,
arenas_i(mib[2])->astats->bstats[mib[4]].nmalloc, uint64_t)

14
src/mutex.c
View File

@ -67,7 +67,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void
malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
lock_prof_data_t *data = &mutex->prof_data;
mutex_prof_data_t *data = &mutex->prof_data;
{//TODO: a smart spin policy
if (!malloc_mutex_trylock(mutex)) {
@ -108,15 +108,21 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) {
}
static void
lock_prof_data_init(lock_prof_data_t *data) {
memset(data, 0, sizeof(lock_prof_data_t));
mutex_prof_data_init(mutex_prof_data_t *data) {
memset(data, 0, sizeof(mutex_prof_data_t));
data->prev_owner = NULL;
}
void
malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex) {
malloc_mutex_assert_owner(tsdn, mutex);
mutex_prof_data_init(&mutex->prof_data);
}
bool
malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
witness_rank_t rank) {
lock_prof_data_init(&mutex->prof_data);
mutex_prof_data_init(&mutex->prof_data);
#ifdef _WIN32
# if _WIN32_WINNT >= 0x0600
InitializeSRWLock(&mutex->lock);

147
src/stats.c
View File

@ -58,20 +58,20 @@ get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) {
}
static void
gen_lock_ctl_str(char *str, const char *prefix, const char *lock,
gen_mutex_ctl_str(char *str, const char *prefix, const char *mutex,
const char *counter) {
sprintf(str, "stats.%s.%s.%s", prefix, lock, counter);
malloc_snprintf(str, 128, "stats.%s.%s.%s", prefix, mutex, counter);
}
static void
read_arena_bin_lock_stats(unsigned arena_ind, unsigned bin_ind,
uint64_t results[NUM_LOCK_PROF_COUNTERS]) {
read_arena_bin_mutex_stats(unsigned arena_ind, unsigned bin_ind,
uint64_t results[NUM_MUTEX_PROF_COUNTERS]) {
char cmd[128];
unsigned i;
for (i = 0; i < NUM_LOCK_PROF_COUNTERS; i++) {
gen_lock_ctl_str(cmd, "arenas.0.bins.0","lock",
lock_counter_names[i]);
for (i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) {
gen_mutex_ctl_str(cmd, "arenas.0.bins.0","mutex",
mutex_counter_names[i]);
CTL_M2_M4_GET(cmd, arena_ind, bin_ind, &results[i], uint64_t);
}
}
@ -147,8 +147,8 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
size_t);
if (json) {
uint64_t lock_stats[NUM_LOCK_PROF_COUNTERS];
read_arena_bin_lock_stats(i, j, lock_stats);
uint64_t mutex_stats[NUM_MUTEX_PROF_COUNTERS];
read_arena_bin_mutex_stats(i, j, mutex_stats);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\t{\n"
@ -170,15 +170,15 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\t\t\"nreslabs\": %"FMTu64",\n"
"\t\t\t\t\t\t\"curslabs\": %zu,\n"
"\t\t\t\t\t\t\"lock\": {\n",
"\t\t\t\t\t\t\"mutex\": {\n",
nreslabs,
curslabs);
for (unsigned k = 0; k < NUM_LOCK_PROF_COUNTERS; k++) {
for (unsigned k = 0; k < NUM_MUTEX_PROF_COUNTERS; k++) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\t\t\t\"%s\": %"FMTu64"%s\n",
lock_counter_names[k], lock_stats[k],
k == NUM_LOCK_PROF_COUNTERS - 1 ? "" : ",");
mutex_counter_names[k], mutex_stats[k],
k == NUM_MUTEX_PROF_COUNTERS - 1 ? "" : ",");
}
malloc_cprintf(write_cb, cbopaque,
@ -207,14 +207,14 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
not_reached();
}
}
/* Output less info for bin locks to save space. */
/* Output less info for bin mutexes to save space. */
uint64_t num_ops, num_wait, max_wait;
CTL_M2_M4_GET("stats.arenas.0.bins.0.lock.num_wait",
CTL_M2_M4_GET("stats.arenas.0.bins.0.mutex.num_wait",
i, j, &num_wait, uint64_t);
CTL_M2_M4_GET(
"stats.arenas.0.bins.0.lock.max_wait_time", i, j,
"stats.arenas.0.bins.0.mutex.max_wait_time", i, j,
&max_wait, uint64_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.lock.num_ops",
CTL_M2_M4_GET("stats.arenas.0.bins.0.mutex.num_ops",
i, j, &num_ops, uint64_t);
char rate[6];
@ -325,38 +325,38 @@ stats_arena_lextents_print(void (*write_cb)(void *, const char *),
}
static void
read_arena_lock_stats(unsigned arena_ind,
uint64_t results[NUM_ARENA_PROF_LOCKS][NUM_LOCK_PROF_COUNTERS]) {
read_arena_mutex_stats(unsigned arena_ind,
uint64_t results[NUM_ARENA_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]) {
char cmd[128];
unsigned i, j;
for (i = 0; i < NUM_ARENA_PROF_LOCKS; i++) {
for (j = 0; j < NUM_LOCK_PROF_COUNTERS; j++) {
gen_lock_ctl_str(cmd, "arenas.0.locks",
arena_lock_names[i], lock_counter_names[j]);
for (i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) {
for (j = 0; j < NUM_MUTEX_PROF_COUNTERS; j++) {
gen_mutex_ctl_str(cmd, "arenas.0.mutexes",
arena_mutex_names[i], mutex_counter_names[j]);
CTL_M2_GET(cmd, arena_ind, &results[i][j], uint64_t);
}
}
}
static void
lock_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque,
const char *name, uint64_t stats[NUM_LOCK_PROF_COUNTERS],
mutex_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque,
const char *name, uint64_t stats[NUM_MUTEX_PROF_COUNTERS],
const char *json_indent, bool last) {
malloc_cprintf(write_cb, cbopaque, "%s\"%s\": {\n", json_indent, name);
for (unsigned i = 0; i < NUM_LOCK_PROF_COUNTERS; i++) {
for (unsigned i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) {
malloc_cprintf(write_cb, cbopaque, "%s\t\"%s\": %"FMTu64"%s\n",
json_indent, lock_counter_names[i], stats[i],
i < (NUM_LOCK_PROF_COUNTERS - 1) ? "," : "");
json_indent, mutex_counter_names[i], stats[i],
i < (NUM_MUTEX_PROF_COUNTERS - 1) ? "," : "");
}
malloc_cprintf(write_cb, cbopaque, "%s}%s\n", json_indent,
last ? "" : ",");
}
static void
lock_stats_output(void (*write_cb)(void *, const char *), void *cbopaque,
const char *name, uint64_t stats[NUM_LOCK_PROF_COUNTERS],
mutex_stats_output(void (*write_cb)(void *, const char *), void *cbopaque,
const char *name, uint64_t stats[NUM_MUTEX_PROF_COUNTERS],
bool first_mutex) {
if (first_mutex) {
/* Print title. */
@ -370,39 +370,39 @@ lock_stats_output(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque, ":%*c",
(int)(19 - strlen(name)), ' ');
for (unsigned i = 0; i < NUM_LOCK_PROF_COUNTERS; i++) {
for (unsigned i = 0; i < NUM_MUTEX_PROF_COUNTERS; i++) {
malloc_cprintf(write_cb, cbopaque, " %16"FMTu64, stats[i]);
}
malloc_cprintf(write_cb, cbopaque, "\n");
}
static void
stats_arena_locks_print(void (*write_cb)(void *, const char *),
stats_arena_mutexes_print(void (*write_cb)(void *, const char *),
void *cbopaque, bool json, bool json_end, unsigned arena_ind) {
uint64_t lock_stats[NUM_ARENA_PROF_LOCKS][NUM_LOCK_PROF_COUNTERS];
read_arena_lock_stats(arena_ind, lock_stats);
uint64_t mutex_stats[NUM_ARENA_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS];
read_arena_mutex_stats(arena_ind, mutex_stats);
/* Output lock stats. */
/* Output mutex stats. */
if (json) {
malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"locks\": {\n");
for (unsigned i = 0; i < NUM_ARENA_PROF_LOCKS; i++) {
lock_stats_output_json(write_cb, cbopaque,
arena_lock_names[i], lock_stats[i],
"\t\t\t\t\t", (i == NUM_ARENA_PROF_LOCKS - 1));
malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"mutexes\": {\n");
for (unsigned i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) {
mutex_stats_output_json(write_cb, cbopaque,
arena_mutex_names[i], mutex_stats[i],
"\t\t\t\t\t", (i == NUM_ARENA_PROF_MUTEXES - 1));
}
malloc_cprintf(write_cb, cbopaque, "\t\t\t\t}%s\n",
json_end ? "" : ",");
} else {
for (unsigned i = 0; i < NUM_ARENA_PROF_LOCKS; i++) {
lock_stats_output(write_cb, cbopaque,
arena_lock_names[i], lock_stats[i], i == 0);
for (unsigned i = 0; i < NUM_ARENA_PROF_MUTEXES; i++) {
mutex_stats_output(write_cb, cbopaque,
arena_mutex_names[i], mutex_stats[i], i == 0);
}
}
}
static void
stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, unsigned i, bool bins, bool large, bool lock) {
bool json, unsigned i, bool bins, bool large, bool mutex) {
unsigned nthreads;
const char *dss;
ssize_t dirty_decay_time, muzzy_decay_time;
@ -625,14 +625,14 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"resident\": %zu%s\n", resident,
(bins || large || lock) ? "," : "");
(bins || large || mutex) ? "," : "");
} else {
malloc_cprintf(write_cb, cbopaque,
"resident: %12zu\n", resident);
}
if (lock) {
stats_arena_locks_print(write_cb, cbopaque, json,
if (mutex) {
stats_arena_mutexes_print(write_cb, cbopaque, json,
!(bins || large), i);
}
if (bins) {
@ -993,15 +993,16 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
}
}
static void read_global_lock_stats(
uint64_t results[NUM_GLOBAL_PROF_LOCKS][NUM_LOCK_PROF_COUNTERS]) {
static void
read_global_mutex_stats(
uint64_t results[NUM_GLOBAL_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS]) {
char cmd[128];
unsigned i, j;
for (i = 0; i < NUM_GLOBAL_PROF_LOCKS; i++) {
for (j = 0; j < NUM_LOCK_PROF_COUNTERS; j++) {
gen_lock_ctl_str(cmd, "locks", global_lock_names[i],
lock_counter_names[j]);
for (i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) {
for (j = 0; j < NUM_MUTEX_PROF_COUNTERS; j++) {
gen_mutex_ctl_str(cmd, "mutexes", global_mutex_names[i],
mutex_counter_names[j]);
CTL_GET(cmd, &results[i][j], uint64_t);
}
}
@ -1010,7 +1011,7 @@ static void read_global_lock_stats(
static void
stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, bool merged, bool destroyed, bool unmerged, bool bins,
bool large, bool lock) {
bool large, bool mutex) {
size_t allocated, active, metadata, resident, mapped, retained;
CTL_GET("stats.allocated", &allocated, size_t);
@ -1020,9 +1021,9 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_GET("stats.mapped", &mapped, size_t);
CTL_GET("stats.retained", &retained, size_t);
uint64_t lock_stats[NUM_GLOBAL_PROF_LOCKS][NUM_LOCK_PROF_COUNTERS];
if (lock) {
read_global_lock_stats(lock_stats);
uint64_t mutex_stats[NUM_GLOBAL_PROF_MUTEXES][NUM_MUTEX_PROF_COUNTERS];
if (mutex) {
read_global_mutex_stats(mutex_stats);
}
if (json) {
@ -1041,14 +1042,14 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
"\t\t\t\"mapped\": %zu,\n", mapped);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"retained\": %zu,\n", retained);
if (lock) {
if (mutex) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"locks\": {\n");
"\t\t\t\"mutexes\": {\n");
for (unsigned i = 0; i < NUM_GLOBAL_PROF_LOCKS; i++) {
lock_stats_output_json(write_cb, cbopaque,
global_lock_names[i], lock_stats[i],
"\t\t\t\t", i == NUM_GLOBAL_PROF_LOCKS - 1);
for (unsigned i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) {
mutex_stats_output_json(write_cb, cbopaque,
global_mutex_names[i], mutex_stats[i],
"\t\t\t\t", i == NUM_GLOBAL_PROF_MUTEXES - 1);
}
malloc_cprintf(write_cb, cbopaque, "\t\t\t}\n");
}
@ -1059,10 +1060,10 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
"Allocated: %zu, active: %zu, metadata: %zu,"
" resident: %zu, mapped: %zu, retained: %zu\n",
allocated, active, metadata, resident, mapped, retained);
if (lock) {
for (unsigned i = 0; i < NUM_GLOBAL_PROF_LOCKS; i++) {
lock_stats_output(write_cb, cbopaque,
global_lock_names[i], lock_stats[i],
if (mutex) {
for (unsigned i = 0; i < NUM_GLOBAL_PROF_MUTEXES; i++) {
mutex_stats_output(write_cb, cbopaque,
global_mutex_names[i], mutex_stats[i],
i == 0);
}
}
@ -1111,7 +1112,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
"\nMerged arenas stats:\n");
}
stats_arena_print(write_cb, cbopaque, json,
MALLCTL_ARENAS_ALL, bins, large, lock);
MALLCTL_ARENAS_ALL, bins, large, mutex);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t}%s\n",
@ -1133,7 +1134,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
}
stats_arena_print(write_cb, cbopaque, json,
MALLCTL_ARENAS_DESTROYED, bins, large,
lock);
mutex);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t}%s\n", unmerged ? "," :
@ -1159,7 +1160,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
}
stats_arena_print(write_cb,
cbopaque, json, i, bins,
large, lock);
large, mutex);
if (json) {
malloc_cprintf(write_cb,
cbopaque,
@ -1192,7 +1193,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool unmerged = config_stats;
bool bins = true;
bool large = true;
bool lock = true;
bool mutex = true;
/*
* Refresh stats, in case mallctl() was called by the application.
@ -1243,7 +1244,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
large = false;
break;
case 'x':
lock = false;
mutex = false;
break;
default:;
}
@ -1264,7 +1265,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
}
if (config_stats) {
stats_print_helper(write_cb, cbopaque, json, merged, destroyed,
unmerged, bins, large, lock);
unmerged, bins, large, mutex);
}
if (json) {