238 lines
6.7 KiB
C
238 lines
6.7 KiB
C
#ifndef JEMALLOC_INTERNAL_ARENA_STATS_H
|
|
#define JEMALLOC_INTERNAL_ARENA_STATS_H
|
|
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/mutex_prof.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
|
|
/*
|
|
* In those architectures that support 64-bit atomics, we use atomic updates for
|
|
* our 64-bit values. Otherwise, we use a plain uint64_t and synchronize
|
|
* externally.
|
|
*/
|
|
#ifdef JEMALLOC_ATOMIC_U64
|
|
typedef atomic_u64_t arena_stats_u64_t;
|
|
#else
|
|
/* Must hold the arena stats mutex while reading atomically. */
|
|
typedef uint64_t arena_stats_u64_t;
|
|
#endif
|
|
|
|
typedef struct arena_stats_large_s arena_stats_large_t;
|
|
struct arena_stats_large_s {
|
|
/*
|
|
* Total number of allocation/deallocation requests served directly by
|
|
* the arena.
|
|
*/
|
|
arena_stats_u64_t nmalloc;
|
|
arena_stats_u64_t ndalloc;
|
|
|
|
/*
|
|
* Number of allocation requests that correspond to this size class.
|
|
* This includes requests served by tcache, though tcache only
|
|
* periodically merges into this counter.
|
|
*/
|
|
arena_stats_u64_t nrequests; /* Partially derived. */
|
|
|
|
/* Current number of allocations of this size class. */
|
|
size_t curlextents; /* Derived. */
|
|
};
|
|
|
|
typedef struct arena_stats_decay_s arena_stats_decay_t;
|
|
struct arena_stats_decay_s {
|
|
/* Total number of purge sweeps. */
|
|
arena_stats_u64_t npurge;
|
|
/* Total number of madvise calls made. */
|
|
arena_stats_u64_t nmadvise;
|
|
/* Total number of pages purged. */
|
|
arena_stats_u64_t purged;
|
|
};
|
|
|
|
/*
|
|
* Arena stats. Note that fields marked "derived" are not directly maintained
|
|
* within the arena code; rather their values are derived during stats merge
|
|
* requests.
|
|
*/
|
|
typedef struct arena_stats_s arena_stats_t;
|
|
struct arena_stats_s {
|
|
#ifndef JEMALLOC_ATOMIC_U64
|
|
malloc_mutex_t mtx;
|
|
#endif
|
|
|
|
/* Number of bytes currently mapped, excluding retained memory. */
|
|
atomic_zu_t mapped; /* Partially derived. */
|
|
|
|
/*
|
|
* Number of unused virtual memory bytes currently retained. Retained
|
|
* bytes are technically mapped (though always decommitted or purged),
|
|
* but they are excluded from the mapped statistic (above).
|
|
*/
|
|
atomic_zu_t retained; /* Derived. */
|
|
|
|
arena_stats_decay_t decay_dirty;
|
|
arena_stats_decay_t decay_muzzy;
|
|
|
|
atomic_zu_t base; /* Derived. */
|
|
atomic_zu_t internal;
|
|
atomic_zu_t resident; /* Derived. */
|
|
atomic_zu_t metadata_thp;
|
|
|
|
atomic_zu_t allocated_large; /* Derived. */
|
|
arena_stats_u64_t nmalloc_large; /* Derived. */
|
|
arena_stats_u64_t ndalloc_large; /* Derived. */
|
|
arena_stats_u64_t nrequests_large; /* Derived. */
|
|
|
|
/* Number of bytes cached in tcache associated with this arena. */
|
|
atomic_zu_t tcache_bytes; /* Derived. */
|
|
|
|
mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes];
|
|
|
|
/* One element for each large size class. */
|
|
arena_stats_large_t lstats[NSIZES - NBINS];
|
|
|
|
/* Arena uptime. */
|
|
nstime_t uptime;
|
|
};
|
|
|
|
static inline bool
|
|
arena_stats_init(UNUSED tsdn_t *tsdn, arena_stats_t *arena_stats) {
|
|
if (config_debug) {
|
|
for (size_t i = 0; i < sizeof(arena_stats_t); i++) {
|
|
assert(((char *)arena_stats)[i] == 0);
|
|
}
|
|
}
|
|
#ifndef JEMALLOC_ATOMIC_U64
|
|
if (malloc_mutex_init(&arena_stats->mtx, "arena_stats",
|
|
WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) {
|
|
return true;
|
|
}
|
|
#endif
|
|
/* Memory is zeroed, so there is no need to clear stats. */
|
|
return false;
|
|
}
|
|
|
|
static inline void
|
|
arena_stats_lock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
|
|
#ifndef JEMALLOC_ATOMIC_U64
|
|
malloc_mutex_lock(tsdn, &arena_stats->mtx);
|
|
#endif
|
|
}
|
|
|
|
static inline void
|
|
arena_stats_unlock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
|
|
#ifndef JEMALLOC_ATOMIC_U64
|
|
malloc_mutex_unlock(tsdn, &arena_stats->mtx);
|
|
#endif
|
|
}
|
|
|
|
static inline uint64_t
|
|
arena_stats_read_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
|
|
arena_stats_u64_t *p) {
|
|
#ifdef JEMALLOC_ATOMIC_U64
|
|
return atomic_load_u64(p, ATOMIC_RELAXED);
|
|
#else
|
|
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
|
|
return *p;
|
|
#endif
|
|
}
|
|
|
|
static inline void
|
|
arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
|
|
arena_stats_u64_t *p, uint64_t x) {
|
|
#ifdef JEMALLOC_ATOMIC_U64
|
|
atomic_fetch_add_u64(p, x, ATOMIC_RELAXED);
|
|
#else
|
|
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
|
|
*p += x;
|
|
#endif
|
|
}
|
|
|
|
UNUSED static inline void
|
|
arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
|
|
arena_stats_u64_t *p, uint64_t x) {
|
|
#ifdef JEMALLOC_ATOMIC_U64
|
|
UNUSED uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED);
|
|
assert(r - x <= r);
|
|
#else
|
|
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
|
|
*p -= x;
|
|
assert(*p + x >= *p);
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Non-atomically sets *dst += src. *dst needs external synchronization.
|
|
* This lets us avoid the cost of a fetch_add when its unnecessary (note that
|
|
* the types here are atomic).
|
|
*/
|
|
static inline void
|
|
arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) {
|
|
#ifdef JEMALLOC_ATOMIC_U64
|
|
uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED);
|
|
atomic_store_u64(dst, src + cur_dst, ATOMIC_RELAXED);
|
|
#else
|
|
*dst += src;
|
|
#endif
|
|
}
|
|
|
|
static inline size_t
|
|
arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) {
|
|
#ifdef JEMALLOC_ATOMIC_U64
|
|
return atomic_load_zu(p, ATOMIC_RELAXED);
|
|
#else
|
|
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
|
|
return atomic_load_zu(p, ATOMIC_RELAXED);
|
|
#endif
|
|
}
|
|
|
|
static inline void
|
|
arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p,
|
|
size_t x) {
|
|
#ifdef JEMALLOC_ATOMIC_U64
|
|
atomic_fetch_add_zu(p, x, ATOMIC_RELAXED);
|
|
#else
|
|
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
|
|
size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
|
|
atomic_store_zu(p, cur + x, ATOMIC_RELAXED);
|
|
#endif
|
|
}
|
|
|
|
static inline void
|
|
arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p,
|
|
size_t x) {
|
|
#ifdef JEMALLOC_ATOMIC_U64
|
|
UNUSED size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED);
|
|
assert(r - x <= r);
|
|
#else
|
|
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
|
|
size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
|
|
atomic_store_zu(p, cur - x, ATOMIC_RELAXED);
|
|
#endif
|
|
}
|
|
|
|
/* Like the _u64 variant, needs an externally synchronized *dst. */
|
|
static inline void
|
|
arena_stats_accum_zu(atomic_zu_t *dst, size_t src) {
|
|
size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);
|
|
atomic_store_zu(dst, src + cur_dst, ATOMIC_RELAXED);
|
|
}
|
|
|
|
static inline void
|
|
arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
|
|
szind_t szind, uint64_t nrequests) {
|
|
arena_stats_lock(tsdn, arena_stats);
|
|
arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind -
|
|
NBINS].nrequests, nrequests);
|
|
arena_stats_unlock(tsdn, arena_stats);
|
|
}
|
|
|
|
static inline void
|
|
arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) {
|
|
arena_stats_lock(tsdn, arena_stats);
|
|
arena_stats_add_zu(tsdn, arena_stats, &arena_stats->mapped, size);
|
|
arena_stats_unlock(tsdn, arena_stats);
|
|
}
|
|
|
|
|
|
#endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */
|