Push locks into arena bins.
For bin-related allocation, protect data structures with bin locks rather than arena locks. Arena locks remain for run allocation/deallocation and other miscellaneous operations. Restructure statistics counters to maintain per bin allocated/nmalloc/ndalloc, but continue to provide arena-wide statistics via aggregation in the ctl code.
This commit is contained in:
Jason Evans
parent
1e0a636c11
commit
86815df9dc
@ -38,7 +38,7 @@
|
||||
.\" @(#)malloc.3 8.1 (Berkeley) 6/4/93
|
||||
.\" $FreeBSD: head/lib/libc/stdlib/malloc.3 182225 2008-08-27 02:00:53Z jasone $
|
||||
.\"
|
||||
.Dd March 4, 2010
|
||||
.Dd March 13, 2010
|
||||
.Dt JEMALLOC 3
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -1134,12 +1134,17 @@ has not been called.
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.small.nmalloc (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of small allocation requests.
|
||||
@roff_stats@Cumulative number of allocation requests served by small bins.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.small.ndalloc (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of small deallocation requests.
|
||||
@roff_stats@Cumulative number of small objects returned to bins.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.small.nrequests (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of small allocation requests.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.medium.allocated (size_t) r-"
|
||||
@ -1149,12 +1154,17 @@ has not been called.
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.medium.nmalloc (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of medium allocation requests.
|
||||
@roff_stats@Cumulative number of allocation requests served by medium bins.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.medium.ndalloc (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of medium deallocation requests.
|
||||
@roff_stats@Cumulative number of medium objects returned to bins.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.medium.nrequests (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of medium allocation requests.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.large.allocated (size_t) r-"
|
||||
@ -1172,6 +1182,21 @@ has not been called.
|
||||
@roff_stats@Cumulative number of large deallocation requests.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.allocated (size_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Current number of bytes allocated by bin.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.nmalloc (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of allocations served by bin.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.ndalloc (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of allocations returned to bin.
|
||||
@roff_stats@.Ed
|
||||
.\"-----------------------------------------------------------------------------
|
||||
@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.nrequests (uint64_t) r-"
|
||||
@roff_stats@.Bd -ragged -offset indent -compact
|
||||
@roff_stats@Cumulative number of allocation requests.
|
||||
|
||||
@ -206,6 +206,14 @@ struct arena_run_s {
|
||||
};
|
||||
|
||||
struct arena_bin_s {
|
||||
/*
|
||||
* All operations on runcur, runs, and stats require that lock be
|
||||
* locked. Run allocation/deallocation are protected by the arena lock,
|
||||
* which may be acquired while holding one or more bin locks, but not
|
||||
* vise versa.
|
||||
*/
|
||||
malloc_mutex_t lock;
|
||||
|
||||
/*
|
||||
* Current run being used to service allocations of this bin's size
|
||||
* class.
|
||||
@ -256,7 +264,10 @@ struct arena_s {
|
||||
/* This arena's index within the arenas array. */
|
||||
unsigned ind;
|
||||
|
||||
/* All operations on this arena require that lock be locked. */
|
||||
/*
|
||||
* All non-bin-related operations on this arena require that lock be
|
||||
* locked.
|
||||
*/
|
||||
malloc_mutex_t lock;
|
||||
|
||||
#ifdef JEMALLOC_STATS
|
||||
@ -459,9 +470,18 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
|
||||
tcache_dalloc(tcache, ptr);
|
||||
else {
|
||||
#endif
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
arena_run_t *run;
|
||||
arena_bin_t *bin;
|
||||
|
||||
run = (arena_run_t *)((uintptr_t)chunk +
|
||||
(uintptr_t)((pageind - ((mapelm->bits &
|
||||
CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) <<
|
||||
PAGE_SHIFT));
|
||||
assert(run->magic == ARENA_RUN_MAGIC);
|
||||
bin = run->bin;
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
arena_dalloc_bin(arena, chunk, ptr, mapelm);
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
#ifdef JEMALLOC_TCACHE
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -33,6 +33,18 @@ struct ctl_arena_stats_s {
|
||||
size_t pdirty;
|
||||
#ifdef JEMALLOC_STATS
|
||||
arena_stats_t astats;
|
||||
|
||||
/* Aggregate stats for small/medium size classes, based on bin stats. */
|
||||
size_t allocated_small;
|
||||
uint64_t nmalloc_small;
|
||||
uint64_t ndalloc_small;
|
||||
uint64_t nrequests_small;
|
||||
|
||||
size_t allocated_medium;
|
||||
uint64_t nmalloc_medium;
|
||||
uint64_t ndalloc_medium;
|
||||
uint64_t nrequests_medium;
|
||||
|
||||
malloc_bin_stats_t *bstats; /* nbins elements. */
|
||||
malloc_large_stats_t *lstats; /* nlclasses elements. */
|
||||
#endif
|
||||
|
||||
@ -31,8 +31,23 @@ struct tcache_bin_stats_s {
|
||||
|
||||
struct malloc_bin_stats_s {
|
||||
/*
|
||||
* Number of allocation requests that corresponded to the size of this
|
||||
* bin.
|
||||
* Current number of bytes allocated, including objects currently
|
||||
* cached by tcache.
|
||||
*/
|
||||
size_t allocated;
|
||||
/*
|
||||
* Total number of allocation/deallocation requests served directly by
|
||||
* the bin. Note that tcache may allocate an object, then recycle it
|
||||
* many times, resulting many increments to nrequests, but only one
|
||||
* each to nmalloc and ndalloc.
|
||||
*/
|
||||
uint64_t nmalloc;
|
||||
uint64_t ndalloc;
|
||||
|
||||
/*
|
||||
* Number of allocation requests that correspond to the size of this
|
||||
* bin. This includes requests served by tcache, though tcache only
|
||||
* periodically merges into this counter.
|
||||
*/
|
||||
uint64_t nrequests;
|
||||
|
||||
@ -87,14 +102,6 @@ struct arena_stats_s {
|
||||
uint64_t purged;
|
||||
|
||||
/* Per-size-category statistics. */
|
||||
size_t allocated_small;
|
||||
uint64_t nmalloc_small;
|
||||
uint64_t ndalloc_small;
|
||||
|
||||
size_t allocated_medium;
|
||||
uint64_t nmalloc_medium;
|
||||
uint64_t ndalloc_medium;
|
||||
|
||||
size_t allocated_large;
|
||||
uint64_t nmalloc_large;
|
||||
uint64_t ndalloc_large;
|
||||
|
||||
@ -64,7 +64,7 @@ extern __thread tcache_t *tcache_tls
|
||||
extern unsigned tcache_gc_incr;
|
||||
|
||||
void tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
|
||||
#ifdef JEMALLOC_PROF
|
||||
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
|
||||
, tcache_t *tcache
|
||||
#endif
|
||||
);
|
||||
@ -130,7 +130,7 @@ tcache_event(tcache_t *tcache)
|
||||
*/
|
||||
tcache_bin_flush(tbin, binind, tbin->ncached -
|
||||
tbin->low_water + (tbin->low_water >> 2)
|
||||
#ifdef JEMALLOC_PROF
|
||||
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
|
||||
, tcache
|
||||
#endif
|
||||
);
|
||||
@ -234,7 +234,7 @@ tcache_dalloc(tcache_t *tcache, void *ptr)
|
||||
tbin = &tcache->tbins[binind];
|
||||
if (tbin->ncached == tbin->ncached_max) {
|
||||
tcache_bin_flush(tbin, binind, (tbin->ncached_max >> 1)
|
||||
#ifdef JEMALLOC_PROF
|
||||
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
|
||||
, tcache
|
||||
#endif
|
||||
);
|
||||
|
||||
@ -769,7 +769,9 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
|
||||
/* No existing runs have any space available. */
|
||||
|
||||
/* Allocate a new run. */
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
run = arena_run_alloc(arena, bin->run_size, false, false);
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
if (run == NULL)
|
||||
return (NULL);
|
||||
|
||||
@ -805,6 +807,21 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
|
||||
return (arena_run_reg_alloc(bin->runcur, bin));
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_PROF
|
||||
void
|
||||
arena_prof_accum(arena_t *arena, uint64_t accumbytes)
|
||||
{
|
||||
|
||||
if (prof_interval != 0) {
|
||||
arena->prof_accumbytes += accumbytes;
|
||||
if (arena->prof_accumbytes >= prof_interval) {
|
||||
prof_idump();
|
||||
arena->prof_accumbytes -= prof_interval;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_TCACHE
|
||||
void
|
||||
arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
|
||||
@ -821,9 +838,11 @@ arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
|
||||
assert(tbin->ncached == 0);
|
||||
|
||||
bin = &arena->bins[binind];
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
#ifdef JEMALLOC_PROF
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
arena_prof_accum(arena, prof_accumbytes);
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
#endif
|
||||
for (i = 0, nfill = (tbin->ncached_max >> 1); i < nfill; i++) {
|
||||
if ((run = bin->runcur) != NULL && run->nfree > 0)
|
||||
@ -836,41 +855,19 @@ arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
|
||||
tbin->avail = ptr;
|
||||
}
|
||||
#ifdef JEMALLOC_STATS
|
||||
bin->stats.nfills++;
|
||||
bin->stats.allocated += (i - tbin->ncached) * bin->reg_size;
|
||||
bin->stats.nmalloc += i;
|
||||
bin->stats.nrequests += tbin->tstats.nrequests;
|
||||
if (bin->reg_size <= small_maxclass) {
|
||||
arena->stats.allocated_small += (i - tbin->ncached) *
|
||||
bin->reg_size;
|
||||
arena->stats.nmalloc_small += tbin->tstats.nrequests;
|
||||
} else {
|
||||
arena->stats.allocated_medium += (i - tbin->ncached) *
|
||||
bin->reg_size;
|
||||
arena->stats.nmalloc_medium += tbin->tstats.nrequests;
|
||||
}
|
||||
bin->stats.nfills++;
|
||||
tbin->tstats.nrequests = 0;
|
||||
#endif
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
tbin->ncached = i;
|
||||
if (tbin->ncached > tbin->high_water)
|
||||
tbin->high_water = tbin->ncached;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_PROF
|
||||
void
|
||||
arena_prof_accum(arena_t *arena, uint64_t accumbytes)
|
||||
{
|
||||
|
||||
if (prof_interval != 0) {
|
||||
arena->prof_accumbytes += accumbytes;
|
||||
if (arena->prof_accumbytes >= prof_interval) {
|
||||
prof_idump();
|
||||
arena->prof_accumbytes -= prof_interval;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Calculate bin->run_size such that it meets the following constraints:
|
||||
*
|
||||
@ -993,33 +990,30 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
|
||||
bin = &arena->bins[binind];
|
||||
size = bin->reg_size;
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
if ((run = bin->runcur) != NULL && run->nfree > 0)
|
||||
ret = arena_run_reg_alloc(run, bin);
|
||||
else
|
||||
ret = arena_bin_malloc_hard(arena, bin);
|
||||
|
||||
if (ret == NULL) {
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_STATS
|
||||
# ifdef JEMALLOC_TCACHE
|
||||
if (isthreaded == false) {
|
||||
# endif
|
||||
bin->stats.allocated += size;
|
||||
bin->stats.nmalloc++;
|
||||
bin->stats.nrequests++;
|
||||
arena->stats.nmalloc_small++;
|
||||
# ifdef JEMALLOC_TCACHE
|
||||
}
|
||||
# endif
|
||||
arena->stats.allocated_small += size;
|
||||
#endif
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
#ifdef JEMALLOC_PROF
|
||||
if (isthreaded == false)
|
||||
if (isthreaded == false) {
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
arena_prof_accum(arena, size);
|
||||
#endif
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (zero == false) {
|
||||
#ifdef JEMALLOC_FILL
|
||||
@ -1048,33 +1042,30 @@ arena_malloc_medium(arena_t *arena, size_t size, bool zero)
|
||||
bin = &arena->bins[binind];
|
||||
assert(bin->reg_size == size);
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
if ((run = bin->runcur) != NULL && run->nfree > 0)
|
||||
ret = arena_run_reg_alloc(run, bin);
|
||||
else
|
||||
ret = arena_bin_malloc_hard(arena, bin);
|
||||
|
||||
if (ret == NULL) {
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_STATS
|
||||
# ifdef JEMALLOC_TCACHE
|
||||
if (isthreaded == false) {
|
||||
# endif
|
||||
bin->stats.allocated += size;
|
||||
bin->stats.nmalloc++;
|
||||
bin->stats.nrequests++;
|
||||
arena->stats.nmalloc_medium++;
|
||||
# ifdef JEMALLOC_TCACHE
|
||||
}
|
||||
# endif
|
||||
arena->stats.allocated_medium += size;
|
||||
#endif
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
#ifdef JEMALLOC_PROF
|
||||
if (isthreaded == false)
|
||||
if (isthreaded == false) {
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
arena_prof_accum(arena, size);
|
||||
#endif
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (zero == false) {
|
||||
#ifdef JEMALLOC_FILL
|
||||
@ -1417,6 +1408,8 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
|
||||
run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT);
|
||||
past = (size_t)(((uintptr_t)run->next - (uintptr_t)1U -
|
||||
(uintptr_t)chunk) >> PAGE_SHIFT) + 1;
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
chunk->ndirty += past - run_ind;
|
||||
arena->ndirty += past - run_ind;
|
||||
for (; run_ind < past; run_ind++) {
|
||||
@ -1428,9 +1421,6 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
|
||||
run->magic = 0;
|
||||
#endif
|
||||
arena_run_dalloc(arena, run, false);
|
||||
#ifdef JEMALLOC_STATS
|
||||
bin->stats.curruns--;
|
||||
#endif
|
||||
|
||||
if (chunk->dirtied == false) {
|
||||
ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty);
|
||||
@ -1440,6 +1430,11 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
|
||||
if (opt_lg_dirty_mult >= 0 && arena->ndirty > chunk_npages &&
|
||||
(arena->nactive >> opt_lg_dirty_mult) < arena->ndirty)
|
||||
arena_purge(arena);
|
||||
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
#ifdef JEMALLOC_STATS
|
||||
bin->stats.curruns--;
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
@ -1508,13 +1503,8 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_STATS
|
||||
if (size <= small_maxclass) {
|
||||
arena->stats.allocated_small -= size;
|
||||
arena->stats.ndalloc_small++;
|
||||
} else {
|
||||
arena->stats.allocated_medium -= size;
|
||||
arena->stats.ndalloc_medium++;
|
||||
}
|
||||
bin->stats.allocated -= size;
|
||||
bin->stats.ndalloc++;
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -1526,6 +1516,7 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
*nactive += arena->nactive;
|
||||
*ndirty += arena->ndirty;
|
||||
|
||||
@ -1533,33 +1524,35 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
|
||||
astats->npurge += arena->stats.npurge;
|
||||
astats->nmadvise += arena->stats.nmadvise;
|
||||
astats->purged += arena->stats.purged;
|
||||
astats->allocated_small += arena->stats.allocated_small;
|
||||
astats->nmalloc_small += arena->stats.nmalloc_small;
|
||||
astats->ndalloc_small += arena->stats.ndalloc_small;
|
||||
astats->allocated_medium += arena->stats.allocated_medium;
|
||||
astats->nmalloc_medium += arena->stats.nmalloc_medium;
|
||||
astats->ndalloc_medium += arena->stats.ndalloc_medium;
|
||||
astats->allocated_large += arena->stats.allocated_large;
|
||||
astats->nmalloc_large += arena->stats.nmalloc_large;
|
||||
astats->ndalloc_large += arena->stats.ndalloc_large;
|
||||
|
||||
for (i = 0; i < nbins; i++) {
|
||||
bstats[i].nrequests += arena->bins[i].stats.nrequests;
|
||||
#ifdef JEMALLOC_TCACHE
|
||||
bstats[i].nfills += arena->bins[i].stats.nfills;
|
||||
bstats[i].nflushes += arena->bins[i].stats.nflushes;
|
||||
#endif
|
||||
bstats[i].nruns += arena->bins[i].stats.nruns;
|
||||
bstats[i].reruns += arena->bins[i].stats.reruns;
|
||||
bstats[i].highruns += arena->bins[i].stats.highruns;
|
||||
bstats[i].curruns += arena->bins[i].stats.curruns;
|
||||
}
|
||||
|
||||
for (i = 0; i < nlclasses; i++) {
|
||||
lstats[i].nrequests += arena->stats.lstats[i].nrequests;
|
||||
lstats[i].highruns += arena->stats.lstats[i].highruns;
|
||||
lstats[i].curruns += arena->stats.lstats[i].curruns;
|
||||
}
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
|
||||
for (i = 0; i < nbins; i++) {
|
||||
arena_bin_t *bin = &arena->bins[i];
|
||||
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
bstats[i].allocated += bin->stats.allocated;
|
||||
bstats[i].nmalloc += bin->stats.nmalloc;
|
||||
bstats[i].ndalloc += bin->stats.ndalloc;
|
||||
bstats[i].nrequests += bin->stats.nrequests;
|
||||
#ifdef JEMALLOC_TCACHE
|
||||
bstats[i].nfills += bin->stats.nfills;
|
||||
bstats[i].nflushes += bin->stats.nflushes;
|
||||
#endif
|
||||
bstats[i].nruns += bin->stats.nruns;
|
||||
bstats[i].reruns += bin->stats.reruns;
|
||||
bstats[i].highruns += bin->stats.highruns;
|
||||
bstats[i].curruns += bin->stats.curruns;
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -1860,6 +1853,8 @@ arena_new(arena_t *arena, unsigned ind)
|
||||
/* (2^n)-spaced tiny bins. */
|
||||
for (; i < ntbins; i++) {
|
||||
bin = &arena->bins[i];
|
||||
if (malloc_mutex_init(&bin->lock))
|
||||
return (true);
|
||||
bin->runcur = NULL;
|
||||
arena_run_tree_new(&bin->runs);
|
||||
|
||||
@ -1876,6 +1871,8 @@ arena_new(arena_t *arena, unsigned ind)
|
||||
/* Quantum-spaced bins. */
|
||||
for (; i < ntbins + nqbins; i++) {
|
||||
bin = &arena->bins[i];
|
||||
if (malloc_mutex_init(&bin->lock))
|
||||
return (true);
|
||||
bin->runcur = NULL;
|
||||
arena_run_tree_new(&bin->runs);
|
||||
|
||||
@ -1891,6 +1888,8 @@ arena_new(arena_t *arena, unsigned ind)
|
||||
/* Cacheline-spaced bins. */
|
||||
for (; i < ntbins + nqbins + ncbins; i++) {
|
||||
bin = &arena->bins[i];
|
||||
if (malloc_mutex_init(&bin->lock))
|
||||
return (true);
|
||||
bin->runcur = NULL;
|
||||
arena_run_tree_new(&bin->runs);
|
||||
|
||||
@ -1907,6 +1906,8 @@ arena_new(arena_t *arena, unsigned ind)
|
||||
/* Subpage-spaced bins. */
|
||||
for (; i < ntbins + nqbins + ncbins + nsbins; i++) {
|
||||
bin = &arena->bins[i];
|
||||
if (malloc_mutex_init(&bin->lock))
|
||||
return (true);
|
||||
bin->runcur = NULL;
|
||||
arena_run_tree_new(&bin->runs);
|
||||
|
||||
@ -1923,6 +1924,8 @@ arena_new(arena_t *arena, unsigned ind)
|
||||
/* Medium bins. */
|
||||
for (; i < nbins; i++) {
|
||||
bin = &arena->bins[i];
|
||||
if (malloc_mutex_init(&bin->lock))
|
||||
return (true);
|
||||
bin->runcur = NULL;
|
||||
arena_run_tree_new(&bin->runs);
|
||||
|
||||
|
||||
@ -24,6 +24,12 @@ const ctl_node_t *n##_index(const size_t *mib, size_t miblen, \
|
||||
static bool ctl_arena_init(ctl_arena_stats_t *astats);
|
||||
#endif
|
||||
static void ctl_arena_clear(ctl_arena_stats_t *astats);
|
||||
#ifdef JEMALLOC_STATS
|
||||
static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats,
|
||||
arena_t *arena);
|
||||
static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,
|
||||
ctl_arena_stats_t *astats);
|
||||
#endif
|
||||
static void ctl_arena_refresh(arena_t *arena, unsigned i);
|
||||
static void ctl_refresh(void);
|
||||
static bool ctl_init(void);
|
||||
@ -131,12 +137,17 @@ CTL_PROTO(stats_huge_ndalloc)
|
||||
CTL_PROTO(stats_arenas_i_small_allocated)
|
||||
CTL_PROTO(stats_arenas_i_small_nmalloc)
|
||||
CTL_PROTO(stats_arenas_i_small_ndalloc)
|
||||
CTL_PROTO(stats_arenas_i_small_nrequests)
|
||||
CTL_PROTO(stats_arenas_i_medium_allocated)
|
||||
CTL_PROTO(stats_arenas_i_medium_nmalloc)
|
||||
CTL_PROTO(stats_arenas_i_medium_ndalloc)
|
||||
CTL_PROTO(stats_arenas_i_medium_nrequests)
|
||||
CTL_PROTO(stats_arenas_i_large_allocated)
|
||||
CTL_PROTO(stats_arenas_i_large_nmalloc)
|
||||
CTL_PROTO(stats_arenas_i_large_ndalloc)
|
||||
CTL_PROTO(stats_arenas_i_bins_j_allocated)
|
||||
CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
|
||||
CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
|
||||
CTL_PROTO(stats_arenas_i_bins_j_nrequests)
|
||||
#ifdef JEMALLOC_TCACHE
|
||||
CTL_PROTO(stats_arenas_i_bins_j_nfills)
|
||||
@ -332,13 +343,15 @@ static const ctl_node_t stats_huge_node[] = {
|
||||
static const ctl_node_t stats_arenas_i_small_node[] = {
|
||||
{NAME("allocated"), CTL(stats_arenas_i_small_allocated)},
|
||||
{NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)},
|
||||
{NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}
|
||||
{NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)},
|
||||
{NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}
|
||||
};
|
||||
|
||||
static const ctl_node_t stats_arenas_i_medium_node[] = {
|
||||
{NAME("allocated"), CTL(stats_arenas_i_medium_allocated)},
|
||||
{NAME("nmalloc"), CTL(stats_arenas_i_medium_nmalloc)},
|
||||
{NAME("ndalloc"), CTL(stats_arenas_i_medium_ndalloc)}
|
||||
{NAME("ndalloc"), CTL(stats_arenas_i_medium_ndalloc)},
|
||||
{NAME("nrequests"), CTL(stats_arenas_i_medium_nrequests)}
|
||||
};
|
||||
|
||||
static const ctl_node_t stats_arenas_i_large_node[] = {
|
||||
@ -348,6 +361,9 @@ static const ctl_node_t stats_arenas_i_large_node[] = {
|
||||
};
|
||||
|
||||
static const ctl_node_t stats_arenas_i_bins_j_node[] = {
|
||||
{NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)},
|
||||
{NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)},
|
||||
{NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)},
|
||||
{NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)},
|
||||
#ifdef JEMALLOC_TCACHE
|
||||
{NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)},
|
||||
@ -485,11 +501,93 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
|
||||
astats->pdirty = 0;
|
||||
#ifdef JEMALLOC_STATS
|
||||
memset(&astats->astats, 0, sizeof(arena_stats_t));
|
||||
astats->allocated_small = 0;
|
||||
astats->nmalloc_small = 0;
|
||||
astats->ndalloc_small = 0;
|
||||
astats->nrequests_small = 0;
|
||||
astats->allocated_medium = 0;
|
||||
astats->nmalloc_medium = 0;
|
||||
astats->ndalloc_medium = 0;
|
||||
astats->nrequests_medium = 0;
|
||||
memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t));
|
||||
memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t));
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_STATS
|
||||
static void
|
||||
ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty,
|
||||
&cstats->astats, cstats->bstats, cstats->lstats);
|
||||
|
||||
for (i = 0; i < mbin0; i++) {
|
||||
cstats->allocated_small += cstats->bstats[i].allocated;
|
||||
cstats->nmalloc_small += cstats->bstats[i].nmalloc;
|
||||
cstats->ndalloc_small += cstats->bstats[i].ndalloc;
|
||||
cstats->nrequests_small += cstats->bstats[i].nrequests;
|
||||
}
|
||||
|
||||
for (; i < nbins; i++) {
|
||||
cstats->allocated_medium += cstats->bstats[i].allocated;
|
||||
cstats->nmalloc_medium += cstats->bstats[i].nmalloc;
|
||||
cstats->ndalloc_medium += cstats->bstats[i].ndalloc;
|
||||
cstats->nrequests_medium += cstats->bstats[i].nrequests;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
sstats->pactive += astats->pactive;
|
||||
sstats->pdirty += astats->pdirty;
|
||||
|
||||
sstats->astats.mapped += astats->astats.mapped;
|
||||
sstats->astats.npurge += astats->astats.npurge;
|
||||
sstats->astats.nmadvise += astats->astats.nmadvise;
|
||||
sstats->astats.purged += astats->astats.purged;
|
||||
|
||||
sstats->allocated_small += astats->allocated_small;
|
||||
sstats->nmalloc_small += astats->nmalloc_small;
|
||||
sstats->ndalloc_small += astats->ndalloc_small;
|
||||
sstats->nrequests_small += astats->nrequests_small;
|
||||
|
||||
sstats->allocated_medium += astats->allocated_medium;
|
||||
sstats->nmalloc_medium += astats->nmalloc_medium;
|
||||
sstats->ndalloc_medium += astats->ndalloc_medium;
|
||||
sstats->nrequests_medium += astats->nrequests_medium;
|
||||
|
||||
sstats->astats.allocated_large += astats->astats.allocated_large;
|
||||
sstats->astats.nmalloc_large += astats->astats.nmalloc_large;
|
||||
sstats->astats.ndalloc_large += astats->astats.ndalloc_large;
|
||||
|
||||
for (i = 0; i < nlclasses; i++) {
|
||||
sstats->lstats[i].nrequests += astats->lstats[i].nrequests;
|
||||
sstats->lstats[i].highruns += astats->lstats[i].highruns;
|
||||
sstats->lstats[i].curruns += astats->lstats[i].curruns;
|
||||
}
|
||||
|
||||
for (i = 0; i < nbins; i++) {
|
||||
sstats->bstats[i].allocated += astats->bstats[i].allocated;
|
||||
sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
|
||||
sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc;
|
||||
sstats->bstats[i].nrequests += astats->bstats[i].nrequests;
|
||||
#ifdef JEMALLOC_TCACHE
|
||||
sstats->bstats[i].nfills += astats->bstats[i].nfills;
|
||||
sstats->bstats[i].nflushes += astats->bstats[i].nflushes;
|
||||
#endif
|
||||
sstats->bstats[i].nruns += astats->bstats[i].nruns;
|
||||
sstats->bstats[i].reruns += astats->bstats[i].reruns;
|
||||
sstats->bstats[i].highruns += astats->bstats[i].highruns;
|
||||
sstats->bstats[i].curruns += astats->bstats[i].curruns;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
static void
|
||||
ctl_arena_refresh(arena_t *arena, unsigned i)
|
||||
{
|
||||
@ -498,13 +596,10 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
|
||||
|
||||
ctl_arena_clear(astats);
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
#ifdef JEMALLOC_STATS
|
||||
arena_stats_merge(arena, &astats->pactive, &astats->pdirty,
|
||||
&astats->astats, astats->bstats, astats->lstats);
|
||||
ctl_arena_stats_amerge(astats, arena);
|
||||
/* Merge into sum stats as well. */
|
||||
arena_stats_merge(arena, &sstats->pactive, &sstats->pdirty,
|
||||
&sstats->astats, sstats->bstats, sstats->lstats);
|
||||
ctl_arena_stats_smerge(sstats, astats);
|
||||
#else
|
||||
astats->pactive += arena->nactive;
|
||||
astats->pdirty += arena->ndirty;
|
||||
@ -512,7 +607,6 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
|
||||
sstats->pactive += arena->nactive;
|
||||
sstats->pdirty += arena->ndirty;
|
||||
#endif
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -553,8 +647,8 @@ ctl_refresh(void)
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_STATS
|
||||
ctl_stats.allocated = ctl_stats.arenas[narenas].astats.allocated_small
|
||||
+ ctl_stats.arenas[narenas].astats.allocated_medium
|
||||
ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small
|
||||
+ ctl_stats.arenas[narenas].allocated_medium
|
||||
+ ctl_stats.arenas[narenas].astats.allocated_large
|
||||
+ ctl_stats.huge.allocated;
|
||||
ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT)
|
||||
@ -1203,17 +1297,21 @@ CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t)
|
||||
CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t)
|
||||
CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_small_allocated,
|
||||
ctl_stats.arenas[mib[2]].astats.allocated_small, size_t)
|
||||
ctl_stats.arenas[mib[2]].allocated_small, size_t)
|
||||
CTL_RO_GEN(stats_arenas_i_small_nmalloc,
|
||||
ctl_stats.arenas[mib[2]].astats.nmalloc_small, uint64_t)
|
||||
ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_small_ndalloc,
|
||||
ctl_stats.arenas[mib[2]].astats.ndalloc_small, uint64_t)
|
||||
ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_small_nrequests,
|
||||
ctl_stats.arenas[mib[2]].nrequests_small, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_medium_allocated,
|
||||
ctl_stats.arenas[mib[2]].astats.allocated_medium, size_t)
|
||||
ctl_stats.arenas[mib[2]].allocated_medium, size_t)
|
||||
CTL_RO_GEN(stats_arenas_i_medium_nmalloc,
|
||||
ctl_stats.arenas[mib[2]].astats.nmalloc_medium, uint64_t)
|
||||
ctl_stats.arenas[mib[2]].nmalloc_medium, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_medium_ndalloc,
|
||||
ctl_stats.arenas[mib[2]].astats.ndalloc_medium, uint64_t)
|
||||
ctl_stats.arenas[mib[2]].ndalloc_medium, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_medium_nrequests,
|
||||
ctl_stats.arenas[mib[2]].nrequests_medium, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_large_allocated,
|
||||
ctl_stats.arenas[mib[2]].astats.allocated_large, size_t)
|
||||
CTL_RO_GEN(stats_arenas_i_large_nmalloc,
|
||||
@ -1221,6 +1319,12 @@ CTL_RO_GEN(stats_arenas_i_large_nmalloc,
|
||||
CTL_RO_GEN(stats_arenas_i_large_ndalloc,
|
||||
ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t)
|
||||
|
||||
CTL_RO_GEN(stats_arenas_i_bins_j_allocated,
|
||||
ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t)
|
||||
CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc,
|
||||
ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc,
|
||||
ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t)
|
||||
CTL_RO_GEN(stats_arenas_i_bins_j_nrequests,
|
||||
ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t)
|
||||
#ifdef JEMALLOC_TCACHE
|
||||
|
||||
@ -160,12 +160,13 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
CTL_GET("config.tcache", &config_tcache, bool);
|
||||
if (config_tcache) {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"bins: bin size regs pgs nrequests "
|
||||
"nfills nflushes newruns reruns maxruns curruns\n");
|
||||
"bins: bin size regs pgs allocated nmalloc"
|
||||
" ndalloc nrequests nfills nflushes newruns"
|
||||
" reruns maxruns curruns\n");
|
||||
} else {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"bins: bin size regs pgs nrequests "
|
||||
"newruns reruns maxruns curruns\n");
|
||||
"bins: bin size regs pgs allocated nmalloc"
|
||||
" ndalloc newruns reruns maxruns curruns\n");
|
||||
}
|
||||
CTL_GET("arenas.nbins", &nbins, unsigned);
|
||||
for (j = 0, gap_start = UINT_MAX; j < nbins; j++) {
|
||||
@ -177,9 +178,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
gap_start = j;
|
||||
} else {
|
||||
unsigned ntbins_, nqbins, ncbins, nsbins;
|
||||
size_t reg_size, run_size;
|
||||
size_t reg_size, run_size, allocated;
|
||||
uint32_t nregs;
|
||||
uint64_t nrequests, nfills, nflushes, reruns;
|
||||
uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes;
|
||||
uint64_t reruns;
|
||||
size_t highruns, curruns;
|
||||
|
||||
if (gap_start != UINT_MAX) {
|
||||
@ -202,9 +204,15 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
CTL_J_GET("arenas.bin.0.size", ®_size, size_t);
|
||||
CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t);
|
||||
CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t);
|
||||
CTL_IJ_GET("stats.arenas.0.bins.0.allocated",
|
||||
&allocated, size_t);
|
||||
CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc",
|
||||
&nmalloc, uint64_t);
|
||||
CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc",
|
||||
&ndalloc, uint64_t);
|
||||
if (config_tcache) {
|
||||
CTL_IJ_GET("stats.arenas.0.bins.0.nrequests",
|
||||
&nrequests, uint64_t);
|
||||
if (config_tcache) {
|
||||
CTL_IJ_GET("stats.arenas.0.bins.0.nfills",
|
||||
&nfills, uint64_t);
|
||||
CTL_IJ_GET("stats.arenas.0.bins.0.nflushes",
|
||||
@ -218,29 +226,32 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
size_t);
|
||||
if (config_tcache) {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"%13u %1s %5zu %4u %3zu %10"PRIu64
|
||||
" %9"PRIu64" %9"PRIu64" %9"PRIu64""
|
||||
" %9"PRIu64" %7zu %7zu\n",
|
||||
"%13u %1s %5zu %4u %3zu %9zu %9"PRIu64
|
||||
" %9"PRIu64" %10"PRIu64" %9"PRIu64
|
||||
" %9"PRIu64" %9"PRIu64" %9"PRIu64
|
||||
" %7zu %7zu\n",
|
||||
j,
|
||||
j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
|
||||
"Q" : j < ntbins_ + nqbins + ncbins ? "C" :
|
||||
j < ntbins_ + nqbins + ncbins + nsbins ? "S"
|
||||
: "M",
|
||||
reg_size, nregs, run_size / pagesize,
|
||||
nrequests, nfills, nflushes, nruns, reruns,
|
||||
highruns, curruns);
|
||||
allocated, nmalloc, ndalloc, nrequests,
|
||||
nfills, nflushes, nruns, reruns, highruns,
|
||||
curruns);
|
||||
} else {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"%13u %1s %5zu %4u %3zu %10"PRIu64
|
||||
" %9"PRIu64" %9"PRIu64" %7zu %7zu\n",
|
||||
"%13u %1s %5zu %4u %3zu %9zu %9"PRIu64
|
||||
" %9"PRIu64" %9"PRIu64" %9"PRIu64
|
||||
" %7zu %7zu\n",
|
||||
j,
|
||||
j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
|
||||
"Q" : j < ntbins_ + nqbins + ncbins ? "C" :
|
||||
j < ntbins_ + nqbins + ncbins + nsbins ? "S"
|
||||
: "M",
|
||||
reg_size, nregs, run_size / pagesize,
|
||||
nrequests, nruns, reruns, highruns,
|
||||
curruns);
|
||||
allocated, nmalloc, ndalloc, nruns, reruns,
|
||||
highruns, curruns);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -305,9 +316,9 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
size_t pagesize, pactive, pdirty, mapped;
|
||||
uint64_t npurge, nmadvise, purged;
|
||||
size_t small_allocated;
|
||||
uint64_t small_nmalloc, small_ndalloc;
|
||||
uint64_t small_nmalloc, small_ndalloc, small_nrequests;
|
||||
size_t medium_allocated;
|
||||
uint64_t medium_nmalloc, medium_ndalloc;
|
||||
uint64_t medium_nmalloc, medium_ndalloc, medium_nrequests;
|
||||
size_t large_allocated;
|
||||
uint64_t large_nmalloc, large_ndalloc;
|
||||
|
||||
@ -325,30 +336,34 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
nmadvise, nmadvise == 1 ? "" : "s", purged);
|
||||
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
" allocated nmalloc ndalloc\n");
|
||||
" allocated nmalloc ndalloc nrequests\n");
|
||||
CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t);
|
||||
CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t);
|
||||
CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t);
|
||||
CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t);
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"small: %12zu %12"PRIu64" %12"PRIu64"\n",
|
||||
small_allocated, small_nmalloc, small_ndalloc);
|
||||
"small: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
|
||||
small_allocated, small_nmalloc, small_ndalloc, small_nrequests);
|
||||
CTL_I_GET("stats.arenas.0.medium.allocated", &medium_allocated, size_t);
|
||||
CTL_I_GET("stats.arenas.0.medium.nmalloc", &medium_nmalloc, uint64_t);
|
||||
CTL_I_GET("stats.arenas.0.medium.ndalloc", &medium_ndalloc, uint64_t);
|
||||
CTL_I_GET("stats.arenas.0.medium.nrequests", &medium_nrequests,
|
||||
uint64_t);
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"medium: %12zu %12"PRIu64" %12"PRIu64"\n",
|
||||
medium_allocated, medium_nmalloc, medium_ndalloc);
|
||||
"medium: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
|
||||
medium_allocated, medium_nmalloc, medium_ndalloc, medium_nrequests);
|
||||
CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t);
|
||||
CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t);
|
||||
CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t);
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"large: %12zu %12"PRIu64" %12"PRIu64"\n",
|
||||
large_allocated, large_nmalloc, large_ndalloc);
|
||||
"large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
|
||||
large_allocated, large_nmalloc, large_ndalloc, large_nmalloc);
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"total: %12zu %12"PRIu64" %12"PRIu64"\n",
|
||||
"total: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
|
||||
small_allocated + medium_allocated + large_allocated,
|
||||
small_nmalloc + medium_nmalloc + large_nmalloc,
|
||||
small_ndalloc + medium_ndalloc + large_ndalloc);
|
||||
small_ndalloc + medium_ndalloc + large_ndalloc,
|
||||
small_nrequests + medium_nrequests + large_nmalloc);
|
||||
malloc_cprintf(write_cb, cbopaque, "active: %12zu\n",
|
||||
pactive * pagesize );
|
||||
CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t);
|
||||
|
||||
@ -45,7 +45,7 @@ tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
|
||||
|
||||
void
|
||||
tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
|
||||
#ifdef JEMALLOC_PROF
|
||||
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
|
||||
, tcache_t *tcache
|
||||
#endif
|
||||
)
|
||||
@ -53,16 +53,29 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
|
||||
void *flush, *deferred, *ptr;
|
||||
unsigned i, nflush, ndeferred;
|
||||
|
||||
assert(rem <= tbin->ncached);
|
||||
|
||||
for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
|
||||
flush = deferred, nflush = ndeferred) {
|
||||
/* Lock the arena associated with the first object. */
|
||||
/* Lock the arena bin associated with the first object. */
|
||||
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush);
|
||||
arena_t *arena = chunk->arena;
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
#ifdef JEMALLOC_PROF
|
||||
arena_bin_t *bin = &arena->bins[binind];
|
||||
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
|
||||
if (arena == tcache->arena) {
|
||||
# ifdef JEMALLOC_PROF
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
arena_prof_accum(arena, tcache->prof_accumbytes);
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
tcache->prof_accumbytes = 0;
|
||||
# endif
|
||||
# ifdef JEMALLOC_STATS
|
||||
bin->stats.nflushes++;
|
||||
bin->stats.nrequests += tbin->tstats.nrequests;
|
||||
tbin->tstats.nrequests = 0;
|
||||
# endif
|
||||
}
|
||||
#endif
|
||||
deferred = NULL;
|
||||
@ -81,31 +94,16 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
|
||||
} else {
|
||||
/*
|
||||
* This object was allocated via a different
|
||||
* arena than the one that is currently locked.
|
||||
* Stash the object, so that it can be handled
|
||||
* in a future pass.
|
||||
* arena bin than the one that is currently
|
||||
* locked. Stash the object, so that it can be
|
||||
* handled in a future pass.
|
||||
*/
|
||||
*(void **)ptr = deferred;
|
||||
deferred = ptr;
|
||||
ndeferred++;
|
||||
}
|
||||
}
|
||||
#ifdef JEMALLOC_STATS
|
||||
arena->bins[binind].stats.nflushes++;
|
||||
{
|
||||
arena_bin_t *bin = &arena->bins[binind];
|
||||
bin->stats.nrequests += tbin->tstats.nrequests;
|
||||
if (bin->reg_size <= small_maxclass) {
|
||||
arena->stats.nmalloc_small +=
|
||||
tbin->tstats.nrequests;
|
||||
} else {
|
||||
arena->stats.nmalloc_medium +=
|
||||
tbin->tstats.nrequests;
|
||||
}
|
||||
tbin->tstats.nrequests = 0;
|
||||
}
|
||||
#endif
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
|
||||
if (flush != NULL) {
|
||||
/*
|
||||
@ -117,6 +115,8 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
|
||||
}
|
||||
|
||||
tbin->ncached = rem;
|
||||
if (tbin->ncached < tbin->low_water)
|
||||
tbin->low_water = tbin->ncached;
|
||||
}
|
||||
|
||||
tcache_t *
|
||||
@ -178,14 +178,14 @@ tcache_destroy(tcache_t *tcache)
|
||||
/* Unlink from list of extant tcaches. */
|
||||
malloc_mutex_lock(&tcache->arena->lock);
|
||||
ql_remove(&tcache->arena->tcache_ql, tcache, link);
|
||||
tcache_stats_merge(tcache, tcache->arena);
|
||||
malloc_mutex_unlock(&tcache->arena->lock);
|
||||
tcache_stats_merge(tcache, tcache->arena);
|
||||
#endif
|
||||
|
||||
for (i = 0; i < nbins; i++) {
|
||||
tcache_bin_t *tbin = &tcache->tbins[i];
|
||||
tcache_bin_flush(tbin, i, 0
|
||||
#ifdef JEMALLOC_PROF
|
||||
#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
|
||||
, tcache
|
||||
#endif
|
||||
);
|
||||
@ -194,16 +194,9 @@ tcache_destroy(tcache_t *tcache)
|
||||
if (tbin->tstats.nrequests != 0) {
|
||||
arena_t *arena = tcache->arena;
|
||||
arena_bin_t *bin = &arena->bins[i];
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
bin->stats.nrequests += tbin->tstats.nrequests;
|
||||
if (bin->reg_size <= small_maxclass) {
|
||||
arena->stats.nmalloc_small +=
|
||||
tbin->tstats.nrequests;
|
||||
} else {
|
||||
arena->stats.nmalloc_medium +=
|
||||
tbin->tstats.nrequests;
|
||||
}
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@ -222,10 +215,14 @@ tcache_destroy(tcache_t *tcache)
|
||||
size_t pageind = (((uintptr_t)tcache - (uintptr_t)chunk) >>
|
||||
PAGE_SHIFT);
|
||||
arena_chunk_map_t *mapelm = &chunk->map[pageind];
|
||||
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
|
||||
(uintptr_t)((pageind - ((mapelm->bits & CHUNK_MAP_PG_MASK)
|
||||
>> CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT));
|
||||
arena_bin_t *bin = run->bin;
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
arena_dalloc_bin(arena, chunk, tcache, mapelm);
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
} else
|
||||
idalloc(tcache);
|
||||
}
|
||||
@ -250,18 +247,12 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
|
||||
unsigned i;
|
||||
|
||||
/* Merge and reset tcache stats. */
|
||||
for (i = 0; i < mbin0; i++) {
|
||||
for (i = 0; i < nbins; i++) {
|
||||
arena_bin_t *bin = &arena->bins[i];
|
||||
tcache_bin_t *tbin = &tcache->tbins[i];
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
bin->stats.nrequests += tbin->tstats.nrequests;
|
||||
arena->stats.nmalloc_small += tbin->tstats.nrequests;
|
||||
tbin->tstats.nrequests = 0;
|
||||
}
|
||||
for (; i < nbins; i++) {
|
||||
arena_bin_t *bin = &arena->bins[i];
|
||||
tcache_bin_t *tbin = &tcache->tbins[i];
|
||||
bin->stats.nrequests += tbin->tstats.nrequests;
|
||||
arena->stats.nmalloc_medium += tbin->tstats.nrequests;
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
tbin->tstats.nrequests = 0;
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user