Push locks into arena bins.

For bin-related allocation, protect data structures with bin locks
rather than arena locks.  Arena locks remain for run
allocation/deallocation and other miscellaneous operations.

Restructure statistics counters to maintain per bin
allocated/nmalloc/ndalloc, but continue to provide arena-wide statistics
via aggregation in the ctl code.
This commit is contained in:
Jason Evans 2010-03-13 20:32:56 -08:00
parent 1e0a636c11
commit 86815df9dc
9 changed files with 368 additions and 191 deletions

View File

@ -38,7 +38,7 @@
.\" @(#)malloc.3 8.1 (Berkeley) 6/4/93
.\" $FreeBSD: head/lib/libc/stdlib/malloc.3 182225 2008-08-27 02:00:53Z jasone $
.\"
.Dd March 4, 2010
.Dd March 13, 2010
.Dt JEMALLOC 3
.Os
.Sh NAME
@ -1134,12 +1134,17 @@ has not been called.
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.small.nmalloc (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of small allocation requests.
@roff_stats@Cumulative number of allocation requests served by small bins.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.small.ndalloc (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of small deallocation requests.
@roff_stats@Cumulative number of small objects returned to bins.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.small.nrequests (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of small allocation requests.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.medium.allocated (size_t) r-"
@ -1149,12 +1154,17 @@ has not been called.
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.medium.nmalloc (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of medium allocation requests.
@roff_stats@Cumulative number of allocation requests served by medium bins.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.medium.ndalloc (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of medium deallocation requests.
@roff_stats@Cumulative number of medium objects returned to bins.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.medium.nrequests (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of medium allocation requests.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.large.allocated (size_t) r-"
@ -1172,6 +1182,21 @@ has not been called.
@roff_stats@Cumulative number of large deallocation requests.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.allocated (size_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Current number of bytes allocated by bin.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.nmalloc (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of allocations served by bin.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.ndalloc (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of allocations returned to bin.
@roff_stats@.Ed
.\"-----------------------------------------------------------------------------
@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.nrequests (uint64_t) r-"
@roff_stats@.Bd -ragged -offset indent -compact
@roff_stats@Cumulative number of allocation requests.

View File

@ -206,6 +206,14 @@ struct arena_run_s {
};
struct arena_bin_s {
/*
* All operations on runcur, runs, and stats require that lock be
* locked. Run allocation/deallocation are protected by the arena lock,
* which may be acquired while holding one or more bin locks, but not
* vise versa.
*/
malloc_mutex_t lock;
/*
* Current run being used to service allocations of this bin's size
* class.
@ -256,7 +264,10 @@ struct arena_s {
/* This arena's index within the arenas array. */
unsigned ind;
/* All operations on this arena require that lock be locked. */
/*
* All non-bin-related operations on this arena require that lock be
* locked.
*/
malloc_mutex_t lock;
#ifdef JEMALLOC_STATS
@ -459,9 +470,18 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
tcache_dalloc(tcache, ptr);
else {
#endif
malloc_mutex_lock(&arena->lock);
arena_run_t *run;
arena_bin_t *bin;
run = (arena_run_t *)((uintptr_t)chunk +
(uintptr_t)((pageind - ((mapelm->bits &
CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) <<
PAGE_SHIFT));
assert(run->magic == ARENA_RUN_MAGIC);
bin = run->bin;
malloc_mutex_lock(&bin->lock);
arena_dalloc_bin(arena, chunk, ptr, mapelm);
malloc_mutex_unlock(&arena->lock);
malloc_mutex_unlock(&bin->lock);
#ifdef JEMALLOC_TCACHE
}
#endif

View File

@ -33,6 +33,18 @@ struct ctl_arena_stats_s {
size_t pdirty;
#ifdef JEMALLOC_STATS
arena_stats_t astats;
/* Aggregate stats for small/medium size classes, based on bin stats. */
size_t allocated_small;
uint64_t nmalloc_small;
uint64_t ndalloc_small;
uint64_t nrequests_small;
size_t allocated_medium;
uint64_t nmalloc_medium;
uint64_t ndalloc_medium;
uint64_t nrequests_medium;
malloc_bin_stats_t *bstats; /* nbins elements. */
malloc_large_stats_t *lstats; /* nlclasses elements. */
#endif

View File

@ -31,8 +31,23 @@ struct tcache_bin_stats_s {
struct malloc_bin_stats_s {
/*
* Number of allocation requests that corresponded to the size of this
* bin.
* Current number of bytes allocated, including objects currently
* cached by tcache.
*/
size_t allocated;
/*
* Total number of allocation/deallocation requests served directly by
* the bin. Note that tcache may allocate an object, then recycle it
* many times, resulting many increments to nrequests, but only one
* each to nmalloc and ndalloc.
*/
uint64_t nmalloc;
uint64_t ndalloc;
/*
* Number of allocation requests that correspond to the size of this
* bin. This includes requests served by tcache, though tcache only
* periodically merges into this counter.
*/
uint64_t nrequests;
@ -87,14 +102,6 @@ struct arena_stats_s {
uint64_t purged;
/* Per-size-category statistics. */
size_t allocated_small;
uint64_t nmalloc_small;
uint64_t ndalloc_small;
size_t allocated_medium;
uint64_t nmalloc_medium;
uint64_t ndalloc_medium;
size_t allocated_large;
uint64_t nmalloc_large;
uint64_t ndalloc_large;

View File

@ -64,7 +64,7 @@ extern __thread tcache_t *tcache_tls
extern unsigned tcache_gc_incr;
void tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
#ifdef JEMALLOC_PROF
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
, tcache_t *tcache
#endif
);
@ -130,7 +130,7 @@ tcache_event(tcache_t *tcache)
*/
tcache_bin_flush(tbin, binind, tbin->ncached -
tbin->low_water + (tbin->low_water >> 2)
#ifdef JEMALLOC_PROF
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
, tcache
#endif
);
@ -234,7 +234,7 @@ tcache_dalloc(tcache_t *tcache, void *ptr)
tbin = &tcache->tbins[binind];
if (tbin->ncached == tbin->ncached_max) {
tcache_bin_flush(tbin, binind, (tbin->ncached_max >> 1)
#ifdef JEMALLOC_PROF
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
, tcache
#endif
);

View File

@ -769,7 +769,9 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
/* No existing runs have any space available. */
/* Allocate a new run. */
malloc_mutex_lock(&arena->lock);
run = arena_run_alloc(arena, bin->run_size, false, false);
malloc_mutex_unlock(&arena->lock);
if (run == NULL)
return (NULL);
@ -805,6 +807,21 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
return (arena_run_reg_alloc(bin->runcur, bin));
}
#ifdef JEMALLOC_PROF
void
arena_prof_accum(arena_t *arena, uint64_t accumbytes)
{
if (prof_interval != 0) {
arena->prof_accumbytes += accumbytes;
if (arena->prof_accumbytes >= prof_interval) {
prof_idump();
arena->prof_accumbytes -= prof_interval;
}
}
}
#endif
#ifdef JEMALLOC_TCACHE
void
arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
@ -821,9 +838,11 @@ arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
assert(tbin->ncached == 0);
bin = &arena->bins[binind];
malloc_mutex_lock(&arena->lock);
malloc_mutex_lock(&bin->lock);
#ifdef JEMALLOC_PROF
malloc_mutex_lock(&arena->lock);
arena_prof_accum(arena, prof_accumbytes);
malloc_mutex_unlock(&arena->lock);
#endif
for (i = 0, nfill = (tbin->ncached_max >> 1); i < nfill; i++) {
if ((run = bin->runcur) != NULL && run->nfree > 0)
@ -836,41 +855,19 @@ arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
tbin->avail = ptr;
}
#ifdef JEMALLOC_STATS
bin->stats.nfills++;
bin->stats.allocated += (i - tbin->ncached) * bin->reg_size;
bin->stats.nmalloc += i;
bin->stats.nrequests += tbin->tstats.nrequests;
if (bin->reg_size <= small_maxclass) {
arena->stats.allocated_small += (i - tbin->ncached) *
bin->reg_size;
arena->stats.nmalloc_small += tbin->tstats.nrequests;
} else {
arena->stats.allocated_medium += (i - tbin->ncached) *
bin->reg_size;
arena->stats.nmalloc_medium += tbin->tstats.nrequests;
}
bin->stats.nfills++;
tbin->tstats.nrequests = 0;
#endif
malloc_mutex_unlock(&arena->lock);
malloc_mutex_unlock(&bin->lock);
tbin->ncached = i;
if (tbin->ncached > tbin->high_water)
tbin->high_water = tbin->ncached;
}
#endif
#ifdef JEMALLOC_PROF
void
arena_prof_accum(arena_t *arena, uint64_t accumbytes)
{
if (prof_interval != 0) {
arena->prof_accumbytes += accumbytes;
if (arena->prof_accumbytes >= prof_interval) {
prof_idump();
arena->prof_accumbytes -= prof_interval;
}
}
}
#endif
/*
* Calculate bin->run_size such that it meets the following constraints:
*
@ -993,33 +990,30 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
bin = &arena->bins[binind];
size = bin->reg_size;
malloc_mutex_lock(&arena->lock);
malloc_mutex_lock(&bin->lock);
if ((run = bin->runcur) != NULL && run->nfree > 0)
ret = arena_run_reg_alloc(run, bin);
else
ret = arena_bin_malloc_hard(arena, bin);
if (ret == NULL) {
malloc_mutex_unlock(&arena->lock);
malloc_mutex_unlock(&bin->lock);
return (NULL);
}
#ifdef JEMALLOC_STATS
# ifdef JEMALLOC_TCACHE
if (isthreaded == false) {
# endif
bin->stats.nrequests++;
arena->stats.nmalloc_small++;
# ifdef JEMALLOC_TCACHE
}
# endif
arena->stats.allocated_small += size;
bin->stats.allocated += size;
bin->stats.nmalloc++;
bin->stats.nrequests++;
#endif
malloc_mutex_unlock(&bin->lock);
#ifdef JEMALLOC_PROF
if (isthreaded == false)
if (isthreaded == false) {
malloc_mutex_lock(&arena->lock);
arena_prof_accum(arena, size);
malloc_mutex_unlock(&arena->lock);
}
#endif
malloc_mutex_unlock(&arena->lock);
if (zero == false) {
#ifdef JEMALLOC_FILL
@ -1048,33 +1042,30 @@ arena_malloc_medium(arena_t *arena, size_t size, bool zero)
bin = &arena->bins[binind];
assert(bin->reg_size == size);
malloc_mutex_lock(&arena->lock);
malloc_mutex_lock(&bin->lock);
if ((run = bin->runcur) != NULL && run->nfree > 0)
ret = arena_run_reg_alloc(run, bin);
else
ret = arena_bin_malloc_hard(arena, bin);
if (ret == NULL) {
malloc_mutex_unlock(&arena->lock);
malloc_mutex_unlock(&bin->lock);
return (NULL);
}
#ifdef JEMALLOC_STATS
# ifdef JEMALLOC_TCACHE
if (isthreaded == false) {
# endif
bin->stats.nrequests++;
arena->stats.nmalloc_medium++;
# ifdef JEMALLOC_TCACHE
}
# endif
arena->stats.allocated_medium += size;
bin->stats.allocated += size;
bin->stats.nmalloc++;
bin->stats.nrequests++;
#endif
malloc_mutex_unlock(&bin->lock);
#ifdef JEMALLOC_PROF
if (isthreaded == false)
if (isthreaded == false) {
malloc_mutex_lock(&arena->lock);
arena_prof_accum(arena, size);
malloc_mutex_unlock(&arena->lock);
}
#endif
malloc_mutex_unlock(&arena->lock);
if (zero == false) {
#ifdef JEMALLOC_FILL
@ -1417,6 +1408,8 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT);
past = (size_t)(((uintptr_t)run->next - (uintptr_t)1U -
(uintptr_t)chunk) >> PAGE_SHIFT) + 1;
malloc_mutex_lock(&arena->lock);
chunk->ndirty += past - run_ind;
arena->ndirty += past - run_ind;
for (; run_ind < past; run_ind++) {
@ -1428,9 +1421,6 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
run->magic = 0;
#endif
arena_run_dalloc(arena, run, false);
#ifdef JEMALLOC_STATS
bin->stats.curruns--;
#endif
if (chunk->dirtied == false) {
ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty);
@ -1440,6 +1430,11 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
if (opt_lg_dirty_mult >= 0 && arena->ndirty > chunk_npages &&
(arena->nactive >> opt_lg_dirty_mult) < arena->ndirty)
arena_purge(arena);
malloc_mutex_unlock(&arena->lock);
#ifdef JEMALLOC_STATS
bin->stats.curruns--;
#endif
}
void
@ -1508,13 +1503,8 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
}
#ifdef JEMALLOC_STATS
if (size <= small_maxclass) {
arena->stats.allocated_small -= size;
arena->stats.ndalloc_small++;
} else {
arena->stats.allocated_medium -= size;
arena->stats.ndalloc_medium++;
}
bin->stats.allocated -= size;
bin->stats.ndalloc++;
#endif
}
@ -1526,6 +1516,7 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
{
unsigned i;
malloc_mutex_lock(&arena->lock);
*nactive += arena->nactive;
*ndirty += arena->ndirty;
@ -1533,33 +1524,35 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
astats->npurge += arena->stats.npurge;
astats->nmadvise += arena->stats.nmadvise;
astats->purged += arena->stats.purged;
astats->allocated_small += arena->stats.allocated_small;
astats->nmalloc_small += arena->stats.nmalloc_small;
astats->ndalloc_small += arena->stats.ndalloc_small;
astats->allocated_medium += arena->stats.allocated_medium;
astats->nmalloc_medium += arena->stats.nmalloc_medium;
astats->ndalloc_medium += arena->stats.ndalloc_medium;
astats->allocated_large += arena->stats.allocated_large;
astats->nmalloc_large += arena->stats.nmalloc_large;
astats->ndalloc_large += arena->stats.ndalloc_large;
for (i = 0; i < nbins; i++) {
bstats[i].nrequests += arena->bins[i].stats.nrequests;
#ifdef JEMALLOC_TCACHE
bstats[i].nfills += arena->bins[i].stats.nfills;
bstats[i].nflushes += arena->bins[i].stats.nflushes;
#endif
bstats[i].nruns += arena->bins[i].stats.nruns;
bstats[i].reruns += arena->bins[i].stats.reruns;
bstats[i].highruns += arena->bins[i].stats.highruns;
bstats[i].curruns += arena->bins[i].stats.curruns;
}
for (i = 0; i < nlclasses; i++) {
lstats[i].nrequests += arena->stats.lstats[i].nrequests;
lstats[i].highruns += arena->stats.lstats[i].highruns;
lstats[i].curruns += arena->stats.lstats[i].curruns;
}
malloc_mutex_unlock(&arena->lock);
for (i = 0; i < nbins; i++) {
arena_bin_t *bin = &arena->bins[i];
malloc_mutex_lock(&bin->lock);
bstats[i].allocated += bin->stats.allocated;
bstats[i].nmalloc += bin->stats.nmalloc;
bstats[i].ndalloc += bin->stats.ndalloc;
bstats[i].nrequests += bin->stats.nrequests;
#ifdef JEMALLOC_TCACHE
bstats[i].nfills += bin->stats.nfills;
bstats[i].nflushes += bin->stats.nflushes;
#endif
bstats[i].nruns += bin->stats.nruns;
bstats[i].reruns += bin->stats.reruns;
bstats[i].highruns += bin->stats.highruns;
bstats[i].curruns += bin->stats.curruns;
malloc_mutex_unlock(&bin->lock);
}
}
#endif
@ -1860,6 +1853,8 @@ arena_new(arena_t *arena, unsigned ind)
/* (2^n)-spaced tiny bins. */
for (; i < ntbins; i++) {
bin = &arena->bins[i];
if (malloc_mutex_init(&bin->lock))
return (true);
bin->runcur = NULL;
arena_run_tree_new(&bin->runs);
@ -1876,6 +1871,8 @@ arena_new(arena_t *arena, unsigned ind)
/* Quantum-spaced bins. */
for (; i < ntbins + nqbins; i++) {
bin = &arena->bins[i];
if (malloc_mutex_init(&bin->lock))
return (true);
bin->runcur = NULL;
arena_run_tree_new(&bin->runs);
@ -1891,6 +1888,8 @@ arena_new(arena_t *arena, unsigned ind)
/* Cacheline-spaced bins. */
for (; i < ntbins + nqbins + ncbins; i++) {
bin = &arena->bins[i];
if (malloc_mutex_init(&bin->lock))
return (true);
bin->runcur = NULL;
arena_run_tree_new(&bin->runs);
@ -1907,6 +1906,8 @@ arena_new(arena_t *arena, unsigned ind)
/* Subpage-spaced bins. */
for (; i < ntbins + nqbins + ncbins + nsbins; i++) {
bin = &arena->bins[i];
if (malloc_mutex_init(&bin->lock))
return (true);
bin->runcur = NULL;
arena_run_tree_new(&bin->runs);
@ -1923,6 +1924,8 @@ arena_new(arena_t *arena, unsigned ind)
/* Medium bins. */
for (; i < nbins; i++) {
bin = &arena->bins[i];
if (malloc_mutex_init(&bin->lock))
return (true);
bin->runcur = NULL;
arena_run_tree_new(&bin->runs);

View File

@ -24,6 +24,12 @@ const ctl_node_t *n##_index(const size_t *mib, size_t miblen, \
static bool ctl_arena_init(ctl_arena_stats_t *astats);
#endif
static void ctl_arena_clear(ctl_arena_stats_t *astats);
#ifdef JEMALLOC_STATS
static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats,
arena_t *arena);
static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,
ctl_arena_stats_t *astats);
#endif
static void ctl_arena_refresh(arena_t *arena, unsigned i);
static void ctl_refresh(void);
static bool ctl_init(void);
@ -131,12 +137,17 @@ CTL_PROTO(stats_huge_ndalloc)
CTL_PROTO(stats_arenas_i_small_allocated)
CTL_PROTO(stats_arenas_i_small_nmalloc)
CTL_PROTO(stats_arenas_i_small_ndalloc)
CTL_PROTO(stats_arenas_i_small_nrequests)
CTL_PROTO(stats_arenas_i_medium_allocated)
CTL_PROTO(stats_arenas_i_medium_nmalloc)
CTL_PROTO(stats_arenas_i_medium_ndalloc)
CTL_PROTO(stats_arenas_i_medium_nrequests)
CTL_PROTO(stats_arenas_i_large_allocated)
CTL_PROTO(stats_arenas_i_large_nmalloc)
CTL_PROTO(stats_arenas_i_large_ndalloc)
CTL_PROTO(stats_arenas_i_bins_j_allocated)
CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
CTL_PROTO(stats_arenas_i_bins_j_nrequests)
#ifdef JEMALLOC_TCACHE
CTL_PROTO(stats_arenas_i_bins_j_nfills)
@ -332,13 +343,15 @@ static const ctl_node_t stats_huge_node[] = {
static const ctl_node_t stats_arenas_i_small_node[] = {
{NAME("allocated"), CTL(stats_arenas_i_small_allocated)},
{NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}
{NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)},
{NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}
};
static const ctl_node_t stats_arenas_i_medium_node[] = {
{NAME("allocated"), CTL(stats_arenas_i_medium_allocated)},
{NAME("nmalloc"), CTL(stats_arenas_i_medium_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_medium_ndalloc)}
{NAME("ndalloc"), CTL(stats_arenas_i_medium_ndalloc)},
{NAME("nrequests"), CTL(stats_arenas_i_medium_nrequests)}
};
static const ctl_node_t stats_arenas_i_large_node[] = {
@ -348,6 +361,9 @@ static const ctl_node_t stats_arenas_i_large_node[] = {
};
static const ctl_node_t stats_arenas_i_bins_j_node[] = {
{NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)},
{NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)},
{NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)},
#ifdef JEMALLOC_TCACHE
{NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)},
@ -485,11 +501,93 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
astats->pdirty = 0;
#ifdef JEMALLOC_STATS
memset(&astats->astats, 0, sizeof(arena_stats_t));
astats->allocated_small = 0;
astats->nmalloc_small = 0;
astats->ndalloc_small = 0;
astats->nrequests_small = 0;
astats->allocated_medium = 0;
astats->nmalloc_medium = 0;
astats->ndalloc_medium = 0;
astats->nrequests_medium = 0;
memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t));
memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t));
#endif
}
#ifdef JEMALLOC_STATS
static void
ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
{
unsigned i;
arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty,
&cstats->astats, cstats->bstats, cstats->lstats);
for (i = 0; i < mbin0; i++) {
cstats->allocated_small += cstats->bstats[i].allocated;
cstats->nmalloc_small += cstats->bstats[i].nmalloc;
cstats->ndalloc_small += cstats->bstats[i].ndalloc;
cstats->nrequests_small += cstats->bstats[i].nrequests;
}
for (; i < nbins; i++) {
cstats->allocated_medium += cstats->bstats[i].allocated;
cstats->nmalloc_medium += cstats->bstats[i].nmalloc;
cstats->ndalloc_medium += cstats->bstats[i].ndalloc;
cstats->nrequests_medium += cstats->bstats[i].nrequests;
}
}
static void
ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
{
unsigned i;
sstats->pactive += astats->pactive;
sstats->pdirty += astats->pdirty;
sstats->astats.mapped += astats->astats.mapped;
sstats->astats.npurge += astats->astats.npurge;
sstats->astats.nmadvise += astats->astats.nmadvise;
sstats->astats.purged += astats->astats.purged;
sstats->allocated_small += astats->allocated_small;
sstats->nmalloc_small += astats->nmalloc_small;
sstats->ndalloc_small += astats->ndalloc_small;
sstats->nrequests_small += astats->nrequests_small;
sstats->allocated_medium += astats->allocated_medium;
sstats->nmalloc_medium += astats->nmalloc_medium;
sstats->ndalloc_medium += astats->ndalloc_medium;
sstats->nrequests_medium += astats->nrequests_medium;
sstats->astats.allocated_large += astats->astats.allocated_large;
sstats->astats.nmalloc_large += astats->astats.nmalloc_large;
sstats->astats.ndalloc_large += astats->astats.ndalloc_large;
for (i = 0; i < nlclasses; i++) {
sstats->lstats[i].nrequests += astats->lstats[i].nrequests;
sstats->lstats[i].highruns += astats->lstats[i].highruns;
sstats->lstats[i].curruns += astats->lstats[i].curruns;
}
for (i = 0; i < nbins; i++) {
sstats->bstats[i].allocated += astats->bstats[i].allocated;
sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc;
sstats->bstats[i].nrequests += astats->bstats[i].nrequests;
#ifdef JEMALLOC_TCACHE
sstats->bstats[i].nfills += astats->bstats[i].nfills;
sstats->bstats[i].nflushes += astats->bstats[i].nflushes;
#endif
sstats->bstats[i].nruns += astats->bstats[i].nruns;
sstats->bstats[i].reruns += astats->bstats[i].reruns;
sstats->bstats[i].highruns += astats->bstats[i].highruns;
sstats->bstats[i].curruns += astats->bstats[i].curruns;
}
}
#endif
static void
ctl_arena_refresh(arena_t *arena, unsigned i)
{
@ -498,13 +596,10 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
ctl_arena_clear(astats);
malloc_mutex_lock(&arena->lock);
#ifdef JEMALLOC_STATS
arena_stats_merge(arena, &astats->pactive, &astats->pdirty,
&astats->astats, astats->bstats, astats->lstats);
ctl_arena_stats_amerge(astats, arena);
/* Merge into sum stats as well. */
arena_stats_merge(arena, &sstats->pactive, &sstats->pdirty,
&sstats->astats, sstats->bstats, sstats->lstats);
ctl_arena_stats_smerge(sstats, astats);
#else
astats->pactive += arena->nactive;
astats->pdirty += arena->ndirty;
@ -512,7 +607,6 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
sstats->pactive += arena->nactive;
sstats->pdirty += arena->ndirty;
#endif
malloc_mutex_unlock(&arena->lock);
}
static void
@ -553,8 +647,8 @@ ctl_refresh(void)
}
#ifdef JEMALLOC_STATS
ctl_stats.allocated = ctl_stats.arenas[narenas].astats.allocated_small
+ ctl_stats.arenas[narenas].astats.allocated_medium
ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small
+ ctl_stats.arenas[narenas].allocated_medium
+ ctl_stats.arenas[narenas].astats.allocated_large
+ ctl_stats.huge.allocated;
ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT)
@ -1203,17 +1297,21 @@ CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t)
CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t)
CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t)
CTL_RO_GEN(stats_arenas_i_small_allocated,
ctl_stats.arenas[mib[2]].astats.allocated_small, size_t)
ctl_stats.arenas[mib[2]].allocated_small, size_t)
CTL_RO_GEN(stats_arenas_i_small_nmalloc,
ctl_stats.arenas[mib[2]].astats.nmalloc_small, uint64_t)
ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t)
CTL_RO_GEN(stats_arenas_i_small_ndalloc,
ctl_stats.arenas[mib[2]].astats.ndalloc_small, uint64_t)
ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t)
CTL_RO_GEN(stats_arenas_i_small_nrequests,
ctl_stats.arenas[mib[2]].nrequests_small, uint64_t)
CTL_RO_GEN(stats_arenas_i_medium_allocated,
ctl_stats.arenas[mib[2]].astats.allocated_medium, size_t)
ctl_stats.arenas[mib[2]].allocated_medium, size_t)
CTL_RO_GEN(stats_arenas_i_medium_nmalloc,
ctl_stats.arenas[mib[2]].astats.nmalloc_medium, uint64_t)
ctl_stats.arenas[mib[2]].nmalloc_medium, uint64_t)
CTL_RO_GEN(stats_arenas_i_medium_ndalloc,
ctl_stats.arenas[mib[2]].astats.ndalloc_medium, uint64_t)
ctl_stats.arenas[mib[2]].ndalloc_medium, uint64_t)
CTL_RO_GEN(stats_arenas_i_medium_nrequests,
ctl_stats.arenas[mib[2]].nrequests_medium, uint64_t)
CTL_RO_GEN(stats_arenas_i_large_allocated,
ctl_stats.arenas[mib[2]].astats.allocated_large, size_t)
CTL_RO_GEN(stats_arenas_i_large_nmalloc,
@ -1221,6 +1319,12 @@ CTL_RO_GEN(stats_arenas_i_large_nmalloc,
CTL_RO_GEN(stats_arenas_i_large_ndalloc,
ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t)
CTL_RO_GEN(stats_arenas_i_bins_j_allocated,
ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t)
CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc,
ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t)
CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc,
ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t)
CTL_RO_GEN(stats_arenas_i_bins_j_nrequests,
ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t)
#ifdef JEMALLOC_TCACHE

View File

@ -160,12 +160,13 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_GET("config.tcache", &config_tcache, bool);
if (config_tcache) {
malloc_cprintf(write_cb, cbopaque,
"bins: bin size regs pgs nrequests "
"nfills nflushes newruns reruns maxruns curruns\n");
"bins: bin size regs pgs allocated nmalloc"
" ndalloc nrequests nfills nflushes newruns"
" reruns maxruns curruns\n");
} else {
malloc_cprintf(write_cb, cbopaque,
"bins: bin size regs pgs nrequests "
"newruns reruns maxruns curruns\n");
"bins: bin size regs pgs allocated nmalloc"
" ndalloc newruns reruns maxruns curruns\n");
}
CTL_GET("arenas.nbins", &nbins, unsigned);
for (j = 0, gap_start = UINT_MAX; j < nbins; j++) {
@ -177,9 +178,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
gap_start = j;
} else {
unsigned ntbins_, nqbins, ncbins, nsbins;
size_t reg_size, run_size;
size_t reg_size, run_size, allocated;
uint32_t nregs;
uint64_t nrequests, nfills, nflushes, reruns;
uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes;
uint64_t reruns;
size_t highruns, curruns;
if (gap_start != UINT_MAX) {
@ -202,9 +204,15 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_J_GET("arenas.bin.0.size", &reg_size, size_t);
CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t);
CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t);
CTL_IJ_GET("stats.arenas.0.bins.0.nrequests",
&nrequests, uint64_t);
CTL_IJ_GET("stats.arenas.0.bins.0.allocated",
&allocated, size_t);
CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc",
&nmalloc, uint64_t);
CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc",
&ndalloc, uint64_t);
if (config_tcache) {
CTL_IJ_GET("stats.arenas.0.bins.0.nrequests",
&nrequests, uint64_t);
CTL_IJ_GET("stats.arenas.0.bins.0.nfills",
&nfills, uint64_t);
CTL_IJ_GET("stats.arenas.0.bins.0.nflushes",
@ -218,29 +226,32 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
size_t);
if (config_tcache) {
malloc_cprintf(write_cb, cbopaque,
"%13u %1s %5zu %4u %3zu %10"PRIu64
" %9"PRIu64" %9"PRIu64" %9"PRIu64""
" %9"PRIu64" %7zu %7zu\n",
"%13u %1s %5zu %4u %3zu %9zu %9"PRIu64
" %9"PRIu64" %10"PRIu64" %9"PRIu64
" %9"PRIu64" %9"PRIu64" %9"PRIu64
" %7zu %7zu\n",
j,
j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
"Q" : j < ntbins_ + nqbins + ncbins ? "C" :
j < ntbins_ + nqbins + ncbins + nsbins ? "S"
: "M",
reg_size, nregs, run_size / pagesize,
nrequests, nfills, nflushes, nruns, reruns,
highruns, curruns);
allocated, nmalloc, ndalloc, nrequests,
nfills, nflushes, nruns, reruns, highruns,
curruns);
} else {
malloc_cprintf(write_cb, cbopaque,
"%13u %1s %5zu %4u %3zu %10"PRIu64
" %9"PRIu64" %9"PRIu64" %7zu %7zu\n",
"%13u %1s %5zu %4u %3zu %9zu %9"PRIu64
" %9"PRIu64" %9"PRIu64" %9"PRIu64
" %7zu %7zu\n",
j,
j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
"Q" : j < ntbins_ + nqbins + ncbins ? "C" :
j < ntbins_ + nqbins + ncbins + nsbins ? "S"
: "M",
reg_size, nregs, run_size / pagesize,
nrequests, nruns, reruns, highruns,
curruns);
allocated, nmalloc, ndalloc, nruns, reruns,
highruns, curruns);
}
}
}
@ -305,9 +316,9 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
size_t pagesize, pactive, pdirty, mapped;
uint64_t npurge, nmadvise, purged;
size_t small_allocated;
uint64_t small_nmalloc, small_ndalloc;
uint64_t small_nmalloc, small_ndalloc, small_nrequests;
size_t medium_allocated;
uint64_t medium_nmalloc, medium_ndalloc;
uint64_t medium_nmalloc, medium_ndalloc, medium_nrequests;
size_t large_allocated;
uint64_t large_nmalloc, large_ndalloc;
@ -325,30 +336,34 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
nmadvise, nmadvise == 1 ? "" : "s", purged);
malloc_cprintf(write_cb, cbopaque,
" allocated nmalloc ndalloc\n");
" allocated nmalloc ndalloc nrequests\n");
CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t);
CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t);
CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t);
CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t);
malloc_cprintf(write_cb, cbopaque,
"small: %12zu %12"PRIu64" %12"PRIu64"\n",
small_allocated, small_nmalloc, small_ndalloc);
"small: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
small_allocated, small_nmalloc, small_ndalloc, small_nrequests);
CTL_I_GET("stats.arenas.0.medium.allocated", &medium_allocated, size_t);
CTL_I_GET("stats.arenas.0.medium.nmalloc", &medium_nmalloc, uint64_t);
CTL_I_GET("stats.arenas.0.medium.ndalloc", &medium_ndalloc, uint64_t);
CTL_I_GET("stats.arenas.0.medium.nrequests", &medium_nrequests,
uint64_t);
malloc_cprintf(write_cb, cbopaque,
"medium: %12zu %12"PRIu64" %12"PRIu64"\n",
medium_allocated, medium_nmalloc, medium_ndalloc);
"medium: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
medium_allocated, medium_nmalloc, medium_ndalloc, medium_nrequests);
CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t);
CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t);
CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t);
malloc_cprintf(write_cb, cbopaque,
"large: %12zu %12"PRIu64" %12"PRIu64"\n",
large_allocated, large_nmalloc, large_ndalloc);
"large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
large_allocated, large_nmalloc, large_ndalloc, large_nmalloc);
malloc_cprintf(write_cb, cbopaque,
"total: %12zu %12"PRIu64" %12"PRIu64"\n",
"total: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
small_allocated + medium_allocated + large_allocated,
small_nmalloc + medium_nmalloc + large_nmalloc,
small_ndalloc + medium_ndalloc + large_ndalloc);
small_ndalloc + medium_ndalloc + large_ndalloc,
small_nrequests + medium_nrequests + large_nmalloc);
malloc_cprintf(write_cb, cbopaque, "active: %12zu\n",
pactive * pagesize );
CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t);

View File

@ -45,7 +45,7 @@ tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
void
tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
#ifdef JEMALLOC_PROF
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
, tcache_t *tcache
#endif
)
@ -53,16 +53,29 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
void *flush, *deferred, *ptr;
unsigned i, nflush, ndeferred;
assert(rem <= tbin->ncached);
for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
flush = deferred, nflush = ndeferred) {
/* Lock the arena associated with the first object. */
/* Lock the arena bin associated with the first object. */
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush);
arena_t *arena = chunk->arena;
malloc_mutex_lock(&arena->lock);
#ifdef JEMALLOC_PROF
arena_bin_t *bin = &arena->bins[binind];
malloc_mutex_lock(&bin->lock);
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
if (arena == tcache->arena) {
# ifdef JEMALLOC_PROF
malloc_mutex_lock(&arena->lock);
arena_prof_accum(arena, tcache->prof_accumbytes);
malloc_mutex_unlock(&arena->lock);
tcache->prof_accumbytes = 0;
# endif
# ifdef JEMALLOC_STATS
bin->stats.nflushes++;
bin->stats.nrequests += tbin->tstats.nrequests;
tbin->tstats.nrequests = 0;
# endif
}
#endif
deferred = NULL;
@ -81,31 +94,16 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
} else {
/*
* This object was allocated via a different
* arena than the one that is currently locked.
* Stash the object, so that it can be handled
* in a future pass.
* arena bin than the one that is currently
* locked. Stash the object, so that it can be
* handled in a future pass.
*/
*(void **)ptr = deferred;
deferred = ptr;
ndeferred++;
}
}
#ifdef JEMALLOC_STATS
arena->bins[binind].stats.nflushes++;
{
arena_bin_t *bin = &arena->bins[binind];
bin->stats.nrequests += tbin->tstats.nrequests;
if (bin->reg_size <= small_maxclass) {
arena->stats.nmalloc_small +=
tbin->tstats.nrequests;
} else {
arena->stats.nmalloc_medium +=
tbin->tstats.nrequests;
}
tbin->tstats.nrequests = 0;
}
#endif
malloc_mutex_unlock(&arena->lock);
malloc_mutex_unlock(&bin->lock);
if (flush != NULL) {
/*
@ -117,6 +115,8 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
}
tbin->ncached = rem;
if (tbin->ncached < tbin->low_water)
tbin->low_water = tbin->ncached;
}
tcache_t *
@ -178,14 +178,14 @@ tcache_destroy(tcache_t *tcache)
/* Unlink from list of extant tcaches. */
malloc_mutex_lock(&tcache->arena->lock);
ql_remove(&tcache->arena->tcache_ql, tcache, link);
tcache_stats_merge(tcache, tcache->arena);
malloc_mutex_unlock(&tcache->arena->lock);
tcache_stats_merge(tcache, tcache->arena);
#endif
for (i = 0; i < nbins; i++) {
tcache_bin_t *tbin = &tcache->tbins[i];
tcache_bin_flush(tbin, i, 0
#ifdef JEMALLOC_PROF
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
, tcache
#endif
);
@ -194,16 +194,9 @@ tcache_destroy(tcache_t *tcache)
if (tbin->tstats.nrequests != 0) {
arena_t *arena = tcache->arena;
arena_bin_t *bin = &arena->bins[i];
malloc_mutex_lock(&arena->lock);
malloc_mutex_lock(&bin->lock);
bin->stats.nrequests += tbin->tstats.nrequests;
if (bin->reg_size <= small_maxclass) {
arena->stats.nmalloc_small +=
tbin->tstats.nrequests;
} else {
arena->stats.nmalloc_medium +=
tbin->tstats.nrequests;
}
malloc_mutex_unlock(&arena->lock);
malloc_mutex_unlock(&bin->lock);
}
#endif
}
@ -222,10 +215,14 @@ tcache_destroy(tcache_t *tcache)
size_t pageind = (((uintptr_t)tcache - (uintptr_t)chunk) >>
PAGE_SHIFT);
arena_chunk_map_t *mapelm = &chunk->map[pageind];
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
(uintptr_t)((pageind - ((mapelm->bits & CHUNK_MAP_PG_MASK)
>> CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT));
arena_bin_t *bin = run->bin;
malloc_mutex_lock(&arena->lock);
malloc_mutex_lock(&bin->lock);
arena_dalloc_bin(arena, chunk, tcache, mapelm);
malloc_mutex_unlock(&arena->lock);
malloc_mutex_unlock(&bin->lock);
} else
idalloc(tcache);
}
@ -250,18 +247,12 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
unsigned i;
/* Merge and reset tcache stats. */
for (i = 0; i < mbin0; i++) {
for (i = 0; i < nbins; i++) {
arena_bin_t *bin = &arena->bins[i];
tcache_bin_t *tbin = &tcache->tbins[i];
malloc_mutex_lock(&bin->lock);
bin->stats.nrequests += tbin->tstats.nrequests;
arena->stats.nmalloc_small += tbin->tstats.nrequests;
tbin->tstats.nrequests = 0;
}
for (; i < nbins; i++) {
arena_bin_t *bin = &arena->bins[i];
tcache_bin_t *tbin = &tcache->tbins[i];
bin->stats.nrequests += tbin->tstats.nrequests;
arena->stats.nmalloc_medium += tbin->tstats.nrequests;
malloc_mutex_unlock(&bin->lock);
tbin->tstats.nrequests = 0;
}
}