From 86815df9dc7d2418a21c87b3dc9747ab42dea73d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 13 Mar 2010 20:32:56 -0800 Subject: [PATCH] Push locks into arena bins. For bin-related allocation, protect data structures with bin locks rather than arena locks. Arena locks remain for run allocation/deallocation and other miscellaneous operations. Restructure statistics counters to maintain per bin allocated/nmalloc/ndalloc, but continue to provide arena-wide statistics via aggregation in the ctl code. --- jemalloc/doc/jemalloc.3.in | 35 ++++- jemalloc/include/jemalloc/internal/arena.h | 26 ++- jemalloc/include/jemalloc/internal/ctl.h | 12 ++ jemalloc/include/jemalloc/internal/stats.h | 27 ++-- jemalloc/include/jemalloc/internal/tcache.h | 6 +- jemalloc/src/arena.c | 165 ++++++++++---------- jemalloc/src/ctl.c | 136 ++++++++++++++-- jemalloc/src/stats.c | 71 +++++---- jemalloc/src/tcache.c | 81 +++++----- 9 files changed, 368 insertions(+), 191 deletions(-) diff --git a/jemalloc/doc/jemalloc.3.in b/jemalloc/doc/jemalloc.3.in index 197e68c3..71f058bf 100644 --- a/jemalloc/doc/jemalloc.3.in +++ b/jemalloc/doc/jemalloc.3.in @@ -38,7 +38,7 @@ .\" @(#)malloc.3 8.1 (Berkeley) 6/4/93 .\" $FreeBSD: head/lib/libc/stdlib/malloc.3 182225 2008-08-27 02:00:53Z jasone $ .\" -.Dd March 4, 2010 +.Dd March 13, 2010 .Dt JEMALLOC 3 .Os .Sh NAME @@ -1134,12 +1134,17 @@ has not been called. .\"----------------------------------------------------------------------------- @roff_stats@.It Sy "stats.arenas..small.nmalloc (uint64_t) r-" @roff_stats@.Bd -ragged -offset indent -compact -@roff_stats@Cumulative number of small allocation requests. +@roff_stats@Cumulative number of allocation requests served by small bins. @roff_stats@.Ed .\"----------------------------------------------------------------------------- @roff_stats@.It Sy "stats.arenas..small.ndalloc (uint64_t) r-" @roff_stats@.Bd -ragged -offset indent -compact -@roff_stats@Cumulative number of small deallocation requests. +@roff_stats@Cumulative number of small objects returned to bins. +@roff_stats@.Ed +.\"----------------------------------------------------------------------------- +@roff_stats@.It Sy "stats.arenas..small.nrequests (uint64_t) r-" +@roff_stats@.Bd -ragged -offset indent -compact +@roff_stats@Cumulative number of small allocation requests. @roff_stats@.Ed .\"----------------------------------------------------------------------------- @roff_stats@.It Sy "stats.arenas..medium.allocated (size_t) r-" @@ -1149,12 +1154,17 @@ has not been called. .\"----------------------------------------------------------------------------- @roff_stats@.It Sy "stats.arenas..medium.nmalloc (uint64_t) r-" @roff_stats@.Bd -ragged -offset indent -compact -@roff_stats@Cumulative number of medium allocation requests. +@roff_stats@Cumulative number of allocation requests served by medium bins. @roff_stats@.Ed .\"----------------------------------------------------------------------------- @roff_stats@.It Sy "stats.arenas..medium.ndalloc (uint64_t) r-" @roff_stats@.Bd -ragged -offset indent -compact -@roff_stats@Cumulative number of medium deallocation requests. +@roff_stats@Cumulative number of medium objects returned to bins. +@roff_stats@.Ed +.\"----------------------------------------------------------------------------- +@roff_stats@.It Sy "stats.arenas..medium.nrequests (uint64_t) r-" +@roff_stats@.Bd -ragged -offset indent -compact +@roff_stats@Cumulative number of medium allocation requests. @roff_stats@.Ed .\"----------------------------------------------------------------------------- @roff_stats@.It Sy "stats.arenas..large.allocated (size_t) r-" @@ -1172,6 +1182,21 @@ has not been called. @roff_stats@Cumulative number of large deallocation requests. @roff_stats@.Ed .\"----------------------------------------------------------------------------- +@roff_stats@.It Sy "stats.arenas..bins..allocated (size_t) r-" +@roff_stats@.Bd -ragged -offset indent -compact +@roff_stats@Current number of bytes allocated by bin. +@roff_stats@.Ed +.\"----------------------------------------------------------------------------- +@roff_stats@.It Sy "stats.arenas..bins..nmalloc (uint64_t) r-" +@roff_stats@.Bd -ragged -offset indent -compact +@roff_stats@Cumulative number of allocations served by bin. +@roff_stats@.Ed +.\"----------------------------------------------------------------------------- +@roff_stats@.It Sy "stats.arenas..bins..ndalloc (uint64_t) r-" +@roff_stats@.Bd -ragged -offset indent -compact +@roff_stats@Cumulative number of allocations returned to bin. +@roff_stats@.Ed +.\"----------------------------------------------------------------------------- @roff_stats@.It Sy "stats.arenas..bins..nrequests (uint64_t) r-" @roff_stats@.Bd -ragged -offset indent -compact @roff_stats@Cumulative number of allocation requests. diff --git a/jemalloc/include/jemalloc/internal/arena.h b/jemalloc/include/jemalloc/internal/arena.h index c6fe823f..06b60cab 100644 --- a/jemalloc/include/jemalloc/internal/arena.h +++ b/jemalloc/include/jemalloc/internal/arena.h @@ -206,6 +206,14 @@ struct arena_run_s { }; struct arena_bin_s { + /* + * All operations on runcur, runs, and stats require that lock be + * locked. Run allocation/deallocation are protected by the arena lock, + * which may be acquired while holding one or more bin locks, but not + * vise versa. + */ + malloc_mutex_t lock; + /* * Current run being used to service allocations of this bin's size * class. @@ -256,7 +264,10 @@ struct arena_s { /* This arena's index within the arenas array. */ unsigned ind; - /* All operations on this arena require that lock be locked. */ + /* + * All non-bin-related operations on this arena require that lock be + * locked. + */ malloc_mutex_t lock; #ifdef JEMALLOC_STATS @@ -459,9 +470,18 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr) tcache_dalloc(tcache, ptr); else { #endif - malloc_mutex_lock(&arena->lock); + arena_run_t *run; + arena_bin_t *bin; + + run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)((pageind - ((mapelm->bits & + CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) << + PAGE_SHIFT)); + assert(run->magic == ARENA_RUN_MAGIC); + bin = run->bin; + malloc_mutex_lock(&bin->lock); arena_dalloc_bin(arena, chunk, ptr, mapelm); - malloc_mutex_unlock(&arena->lock); + malloc_mutex_unlock(&bin->lock); #ifdef JEMALLOC_TCACHE } #endif diff --git a/jemalloc/include/jemalloc/internal/ctl.h b/jemalloc/include/jemalloc/internal/ctl.h index d5b128b8..9a39e14f 100644 --- a/jemalloc/include/jemalloc/internal/ctl.h +++ b/jemalloc/include/jemalloc/internal/ctl.h @@ -33,6 +33,18 @@ struct ctl_arena_stats_s { size_t pdirty; #ifdef JEMALLOC_STATS arena_stats_t astats; + + /* Aggregate stats for small/medium size classes, based on bin stats. */ + size_t allocated_small; + uint64_t nmalloc_small; + uint64_t ndalloc_small; + uint64_t nrequests_small; + + size_t allocated_medium; + uint64_t nmalloc_medium; + uint64_t ndalloc_medium; + uint64_t nrequests_medium; + malloc_bin_stats_t *bstats; /* nbins elements. */ malloc_large_stats_t *lstats; /* nlclasses elements. */ #endif diff --git a/jemalloc/include/jemalloc/internal/stats.h b/jemalloc/include/jemalloc/internal/stats.h index 8e7f95ae..47701e61 100644 --- a/jemalloc/include/jemalloc/internal/stats.h +++ b/jemalloc/include/jemalloc/internal/stats.h @@ -31,8 +31,23 @@ struct tcache_bin_stats_s { struct malloc_bin_stats_s { /* - * Number of allocation requests that corresponded to the size of this - * bin. + * Current number of bytes allocated, including objects currently + * cached by tcache. + */ + size_t allocated; + /* + * Total number of allocation/deallocation requests served directly by + * the bin. Note that tcache may allocate an object, then recycle it + * many times, resulting many increments to nrequests, but only one + * each to nmalloc and ndalloc. + */ + uint64_t nmalloc; + uint64_t ndalloc; + + /* + * Number of allocation requests that correspond to the size of this + * bin. This includes requests served by tcache, though tcache only + * periodically merges into this counter. */ uint64_t nrequests; @@ -87,14 +102,6 @@ struct arena_stats_s { uint64_t purged; /* Per-size-category statistics. */ - size_t allocated_small; - uint64_t nmalloc_small; - uint64_t ndalloc_small; - - size_t allocated_medium; - uint64_t nmalloc_medium; - uint64_t ndalloc_medium; - size_t allocated_large; uint64_t nmalloc_large; uint64_t ndalloc_large; diff --git a/jemalloc/include/jemalloc/internal/tcache.h b/jemalloc/include/jemalloc/internal/tcache.h index fa238bf3..ad30b7c5 100644 --- a/jemalloc/include/jemalloc/internal/tcache.h +++ b/jemalloc/include/jemalloc/internal/tcache.h @@ -64,7 +64,7 @@ extern __thread tcache_t *tcache_tls extern unsigned tcache_gc_incr; void tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem -#ifdef JEMALLOC_PROF +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) , tcache_t *tcache #endif ); @@ -130,7 +130,7 @@ tcache_event(tcache_t *tcache) */ tcache_bin_flush(tbin, binind, tbin->ncached - tbin->low_water + (tbin->low_water >> 2) -#ifdef JEMALLOC_PROF +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) , tcache #endif ); @@ -234,7 +234,7 @@ tcache_dalloc(tcache_t *tcache, void *ptr) tbin = &tcache->tbins[binind]; if (tbin->ncached == tbin->ncached_max) { tcache_bin_flush(tbin, binind, (tbin->ncached_max >> 1) -#ifdef JEMALLOC_PROF +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) , tcache #endif ); diff --git a/jemalloc/src/arena.c b/jemalloc/src/arena.c index e7ecb437..3981cbb3 100644 --- a/jemalloc/src/arena.c +++ b/jemalloc/src/arena.c @@ -769,7 +769,9 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) /* No existing runs have any space available. */ /* Allocate a new run. */ + malloc_mutex_lock(&arena->lock); run = arena_run_alloc(arena, bin->run_size, false, false); + malloc_mutex_unlock(&arena->lock); if (run == NULL) return (NULL); @@ -805,6 +807,21 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) return (arena_run_reg_alloc(bin->runcur, bin)); } +#ifdef JEMALLOC_PROF +void +arena_prof_accum(arena_t *arena, uint64_t accumbytes) +{ + + if (prof_interval != 0) { + arena->prof_accumbytes += accumbytes; + if (arena->prof_accumbytes >= prof_interval) { + prof_idump(); + arena->prof_accumbytes -= prof_interval; + } + } +} +#endif + #ifdef JEMALLOC_TCACHE void arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind @@ -821,9 +838,11 @@ arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind assert(tbin->ncached == 0); bin = &arena->bins[binind]; - malloc_mutex_lock(&arena->lock); + malloc_mutex_lock(&bin->lock); #ifdef JEMALLOC_PROF + malloc_mutex_lock(&arena->lock); arena_prof_accum(arena, prof_accumbytes); + malloc_mutex_unlock(&arena->lock); #endif for (i = 0, nfill = (tbin->ncached_max >> 1); i < nfill; i++) { if ((run = bin->runcur) != NULL && run->nfree > 0) @@ -836,41 +855,19 @@ arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind tbin->avail = ptr; } #ifdef JEMALLOC_STATS - bin->stats.nfills++; + bin->stats.allocated += (i - tbin->ncached) * bin->reg_size; + bin->stats.nmalloc += i; bin->stats.nrequests += tbin->tstats.nrequests; - if (bin->reg_size <= small_maxclass) { - arena->stats.allocated_small += (i - tbin->ncached) * - bin->reg_size; - arena->stats.nmalloc_small += tbin->tstats.nrequests; - } else { - arena->stats.allocated_medium += (i - tbin->ncached) * - bin->reg_size; - arena->stats.nmalloc_medium += tbin->tstats.nrequests; - } + bin->stats.nfills++; tbin->tstats.nrequests = 0; #endif - malloc_mutex_unlock(&arena->lock); + malloc_mutex_unlock(&bin->lock); tbin->ncached = i; if (tbin->ncached > tbin->high_water) tbin->high_water = tbin->ncached; } #endif -#ifdef JEMALLOC_PROF -void -arena_prof_accum(arena_t *arena, uint64_t accumbytes) -{ - - if (prof_interval != 0) { - arena->prof_accumbytes += accumbytes; - if (arena->prof_accumbytes >= prof_interval) { - prof_idump(); - arena->prof_accumbytes -= prof_interval; - } - } -} -#endif - /* * Calculate bin->run_size such that it meets the following constraints: * @@ -993,33 +990,30 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) bin = &arena->bins[binind]; size = bin->reg_size; - malloc_mutex_lock(&arena->lock); + malloc_mutex_lock(&bin->lock); if ((run = bin->runcur) != NULL && run->nfree > 0) ret = arena_run_reg_alloc(run, bin); else ret = arena_bin_malloc_hard(arena, bin); if (ret == NULL) { - malloc_mutex_unlock(&arena->lock); + malloc_mutex_unlock(&bin->lock); return (NULL); } #ifdef JEMALLOC_STATS -# ifdef JEMALLOC_TCACHE - if (isthreaded == false) { -# endif - bin->stats.nrequests++; - arena->stats.nmalloc_small++; -# ifdef JEMALLOC_TCACHE - } -# endif - arena->stats.allocated_small += size; + bin->stats.allocated += size; + bin->stats.nmalloc++; + bin->stats.nrequests++; #endif + malloc_mutex_unlock(&bin->lock); #ifdef JEMALLOC_PROF - if (isthreaded == false) + if (isthreaded == false) { + malloc_mutex_lock(&arena->lock); arena_prof_accum(arena, size); + malloc_mutex_unlock(&arena->lock); + } #endif - malloc_mutex_unlock(&arena->lock); if (zero == false) { #ifdef JEMALLOC_FILL @@ -1048,33 +1042,30 @@ arena_malloc_medium(arena_t *arena, size_t size, bool zero) bin = &arena->bins[binind]; assert(bin->reg_size == size); - malloc_mutex_lock(&arena->lock); + malloc_mutex_lock(&bin->lock); if ((run = bin->runcur) != NULL && run->nfree > 0) ret = arena_run_reg_alloc(run, bin); else ret = arena_bin_malloc_hard(arena, bin); if (ret == NULL) { - malloc_mutex_unlock(&arena->lock); + malloc_mutex_unlock(&bin->lock); return (NULL); } #ifdef JEMALLOC_STATS -# ifdef JEMALLOC_TCACHE - if (isthreaded == false) { -# endif - bin->stats.nrequests++; - arena->stats.nmalloc_medium++; -# ifdef JEMALLOC_TCACHE - } -# endif - arena->stats.allocated_medium += size; + bin->stats.allocated += size; + bin->stats.nmalloc++; + bin->stats.nrequests++; #endif + malloc_mutex_unlock(&bin->lock); #ifdef JEMALLOC_PROF - if (isthreaded == false) + if (isthreaded == false) { + malloc_mutex_lock(&arena->lock); arena_prof_accum(arena, size); + malloc_mutex_unlock(&arena->lock); + } #endif - malloc_mutex_unlock(&arena->lock); if (zero == false) { #ifdef JEMALLOC_FILL @@ -1417,6 +1408,8 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); past = (size_t)(((uintptr_t)run->next - (uintptr_t)1U - (uintptr_t)chunk) >> PAGE_SHIFT) + 1; + + malloc_mutex_lock(&arena->lock); chunk->ndirty += past - run_ind; arena->ndirty += past - run_ind; for (; run_ind < past; run_ind++) { @@ -1428,9 +1421,6 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, run->magic = 0; #endif arena_run_dalloc(arena, run, false); -#ifdef JEMALLOC_STATS - bin->stats.curruns--; -#endif if (chunk->dirtied == false) { ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); @@ -1440,6 +1430,11 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, if (opt_lg_dirty_mult >= 0 && arena->ndirty > chunk_npages && (arena->nactive >> opt_lg_dirty_mult) < arena->ndirty) arena_purge(arena); + + malloc_mutex_unlock(&arena->lock); +#ifdef JEMALLOC_STATS + bin->stats.curruns--; +#endif } void @@ -1508,13 +1503,8 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, } #ifdef JEMALLOC_STATS - if (size <= small_maxclass) { - arena->stats.allocated_small -= size; - arena->stats.ndalloc_small++; - } else { - arena->stats.allocated_medium -= size; - arena->stats.ndalloc_medium++; - } + bin->stats.allocated -= size; + bin->stats.ndalloc++; #endif } @@ -1526,6 +1516,7 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, { unsigned i; + malloc_mutex_lock(&arena->lock); *nactive += arena->nactive; *ndirty += arena->ndirty; @@ -1533,33 +1524,35 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, astats->npurge += arena->stats.npurge; astats->nmadvise += arena->stats.nmadvise; astats->purged += arena->stats.purged; - astats->allocated_small += arena->stats.allocated_small; - astats->nmalloc_small += arena->stats.nmalloc_small; - astats->ndalloc_small += arena->stats.ndalloc_small; - astats->allocated_medium += arena->stats.allocated_medium; - astats->nmalloc_medium += arena->stats.nmalloc_medium; - astats->ndalloc_medium += arena->stats.ndalloc_medium; astats->allocated_large += arena->stats.allocated_large; astats->nmalloc_large += arena->stats.nmalloc_large; astats->ndalloc_large += arena->stats.ndalloc_large; - for (i = 0; i < nbins; i++) { - bstats[i].nrequests += arena->bins[i].stats.nrequests; -#ifdef JEMALLOC_TCACHE - bstats[i].nfills += arena->bins[i].stats.nfills; - bstats[i].nflushes += arena->bins[i].stats.nflushes; -#endif - bstats[i].nruns += arena->bins[i].stats.nruns; - bstats[i].reruns += arena->bins[i].stats.reruns; - bstats[i].highruns += arena->bins[i].stats.highruns; - bstats[i].curruns += arena->bins[i].stats.curruns; - } - for (i = 0; i < nlclasses; i++) { lstats[i].nrequests += arena->stats.lstats[i].nrequests; lstats[i].highruns += arena->stats.lstats[i].highruns; lstats[i].curruns += arena->stats.lstats[i].curruns; } + malloc_mutex_unlock(&arena->lock); + + for (i = 0; i < nbins; i++) { + arena_bin_t *bin = &arena->bins[i]; + + malloc_mutex_lock(&bin->lock); + bstats[i].allocated += bin->stats.allocated; + bstats[i].nmalloc += bin->stats.nmalloc; + bstats[i].ndalloc += bin->stats.ndalloc; + bstats[i].nrequests += bin->stats.nrequests; +#ifdef JEMALLOC_TCACHE + bstats[i].nfills += bin->stats.nfills; + bstats[i].nflushes += bin->stats.nflushes; +#endif + bstats[i].nruns += bin->stats.nruns; + bstats[i].reruns += bin->stats.reruns; + bstats[i].highruns += bin->stats.highruns; + bstats[i].curruns += bin->stats.curruns; + malloc_mutex_unlock(&bin->lock); + } } #endif @@ -1860,6 +1853,8 @@ arena_new(arena_t *arena, unsigned ind) /* (2^n)-spaced tiny bins. */ for (; i < ntbins; i++) { bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); bin->runcur = NULL; arena_run_tree_new(&bin->runs); @@ -1876,6 +1871,8 @@ arena_new(arena_t *arena, unsigned ind) /* Quantum-spaced bins. */ for (; i < ntbins + nqbins; i++) { bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); bin->runcur = NULL; arena_run_tree_new(&bin->runs); @@ -1891,6 +1888,8 @@ arena_new(arena_t *arena, unsigned ind) /* Cacheline-spaced bins. */ for (; i < ntbins + nqbins + ncbins; i++) { bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); bin->runcur = NULL; arena_run_tree_new(&bin->runs); @@ -1907,6 +1906,8 @@ arena_new(arena_t *arena, unsigned ind) /* Subpage-spaced bins. */ for (; i < ntbins + nqbins + ncbins + nsbins; i++) { bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); bin->runcur = NULL; arena_run_tree_new(&bin->runs); @@ -1923,6 +1924,8 @@ arena_new(arena_t *arena, unsigned ind) /* Medium bins. */ for (; i < nbins; i++) { bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); bin->runcur = NULL; arena_run_tree_new(&bin->runs); diff --git a/jemalloc/src/ctl.c b/jemalloc/src/ctl.c index 28e9368c..1644f73e 100644 --- a/jemalloc/src/ctl.c +++ b/jemalloc/src/ctl.c @@ -24,6 +24,12 @@ const ctl_node_t *n##_index(const size_t *mib, size_t miblen, \ static bool ctl_arena_init(ctl_arena_stats_t *astats); #endif static void ctl_arena_clear(ctl_arena_stats_t *astats); +#ifdef JEMALLOC_STATS +static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, + arena_t *arena); +static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, + ctl_arena_stats_t *astats); +#endif static void ctl_arena_refresh(arena_t *arena, unsigned i); static void ctl_refresh(void); static bool ctl_init(void); @@ -131,12 +137,17 @@ CTL_PROTO(stats_huge_ndalloc) CTL_PROTO(stats_arenas_i_small_allocated) CTL_PROTO(stats_arenas_i_small_nmalloc) CTL_PROTO(stats_arenas_i_small_ndalloc) +CTL_PROTO(stats_arenas_i_small_nrequests) CTL_PROTO(stats_arenas_i_medium_allocated) CTL_PROTO(stats_arenas_i_medium_nmalloc) CTL_PROTO(stats_arenas_i_medium_ndalloc) +CTL_PROTO(stats_arenas_i_medium_nrequests) CTL_PROTO(stats_arenas_i_large_allocated) CTL_PROTO(stats_arenas_i_large_nmalloc) CTL_PROTO(stats_arenas_i_large_ndalloc) +CTL_PROTO(stats_arenas_i_bins_j_allocated) +CTL_PROTO(stats_arenas_i_bins_j_nmalloc) +CTL_PROTO(stats_arenas_i_bins_j_ndalloc) CTL_PROTO(stats_arenas_i_bins_j_nrequests) #ifdef JEMALLOC_TCACHE CTL_PROTO(stats_arenas_i_bins_j_nfills) @@ -332,13 +343,15 @@ static const ctl_node_t stats_huge_node[] = { static const ctl_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)} + {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} }; static const ctl_node_t stats_arenas_i_medium_node[] = { {NAME("allocated"), CTL(stats_arenas_i_medium_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_medium_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_medium_ndalloc)} + {NAME("ndalloc"), CTL(stats_arenas_i_medium_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_medium_nrequests)} }; static const ctl_node_t stats_arenas_i_large_node[] = { @@ -348,6 +361,9 @@ static const ctl_node_t stats_arenas_i_large_node[] = { }; static const ctl_node_t stats_arenas_i_bins_j_node[] = { + {NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, #ifdef JEMALLOC_TCACHE {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, @@ -485,11 +501,93 @@ ctl_arena_clear(ctl_arena_stats_t *astats) astats->pdirty = 0; #ifdef JEMALLOC_STATS memset(&astats->astats, 0, sizeof(arena_stats_t)); + astats->allocated_small = 0; + astats->nmalloc_small = 0; + astats->ndalloc_small = 0; + astats->nrequests_small = 0; + astats->allocated_medium = 0; + astats->nmalloc_medium = 0; + astats->ndalloc_medium = 0; + astats->nrequests_medium = 0; memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t)); memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); #endif } +#ifdef JEMALLOC_STATS +static void +ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) +{ + unsigned i; + + arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty, + &cstats->astats, cstats->bstats, cstats->lstats); + + for (i = 0; i < mbin0; i++) { + cstats->allocated_small += cstats->bstats[i].allocated; + cstats->nmalloc_small += cstats->bstats[i].nmalloc; + cstats->ndalloc_small += cstats->bstats[i].ndalloc; + cstats->nrequests_small += cstats->bstats[i].nrequests; + } + + for (; i < nbins; i++) { + cstats->allocated_medium += cstats->bstats[i].allocated; + cstats->nmalloc_medium += cstats->bstats[i].nmalloc; + cstats->ndalloc_medium += cstats->bstats[i].ndalloc; + cstats->nrequests_medium += cstats->bstats[i].nrequests; + } +} + +static void +ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) +{ + unsigned i; + + sstats->pactive += astats->pactive; + sstats->pdirty += astats->pdirty; + + sstats->astats.mapped += astats->astats.mapped; + sstats->astats.npurge += astats->astats.npurge; + sstats->astats.nmadvise += astats->astats.nmadvise; + sstats->astats.purged += astats->astats.purged; + + sstats->allocated_small += astats->allocated_small; + sstats->nmalloc_small += astats->nmalloc_small; + sstats->ndalloc_small += astats->ndalloc_small; + sstats->nrequests_small += astats->nrequests_small; + + sstats->allocated_medium += astats->allocated_medium; + sstats->nmalloc_medium += astats->nmalloc_medium; + sstats->ndalloc_medium += astats->ndalloc_medium; + sstats->nrequests_medium += astats->nrequests_medium; + + sstats->astats.allocated_large += astats->astats.allocated_large; + sstats->astats.nmalloc_large += astats->astats.nmalloc_large; + sstats->astats.ndalloc_large += astats->astats.ndalloc_large; + + for (i = 0; i < nlclasses; i++) { + sstats->lstats[i].nrequests += astats->lstats[i].nrequests; + sstats->lstats[i].highruns += astats->lstats[i].highruns; + sstats->lstats[i].curruns += astats->lstats[i].curruns; + } + + for (i = 0; i < nbins; i++) { + sstats->bstats[i].allocated += astats->bstats[i].allocated; + sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; + sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; + sstats->bstats[i].nrequests += astats->bstats[i].nrequests; +#ifdef JEMALLOC_TCACHE + sstats->bstats[i].nfills += astats->bstats[i].nfills; + sstats->bstats[i].nflushes += astats->bstats[i].nflushes; +#endif + sstats->bstats[i].nruns += astats->bstats[i].nruns; + sstats->bstats[i].reruns += astats->bstats[i].reruns; + sstats->bstats[i].highruns += astats->bstats[i].highruns; + sstats->bstats[i].curruns += astats->bstats[i].curruns; + } +} +#endif + static void ctl_arena_refresh(arena_t *arena, unsigned i) { @@ -498,13 +596,10 @@ ctl_arena_refresh(arena_t *arena, unsigned i) ctl_arena_clear(astats); - malloc_mutex_lock(&arena->lock); #ifdef JEMALLOC_STATS - arena_stats_merge(arena, &astats->pactive, &astats->pdirty, - &astats->astats, astats->bstats, astats->lstats); + ctl_arena_stats_amerge(astats, arena); /* Merge into sum stats as well. */ - arena_stats_merge(arena, &sstats->pactive, &sstats->pdirty, - &sstats->astats, sstats->bstats, sstats->lstats); + ctl_arena_stats_smerge(sstats, astats); #else astats->pactive += arena->nactive; astats->pdirty += arena->ndirty; @@ -512,7 +607,6 @@ ctl_arena_refresh(arena_t *arena, unsigned i) sstats->pactive += arena->nactive; sstats->pdirty += arena->ndirty; #endif - malloc_mutex_unlock(&arena->lock); } static void @@ -553,8 +647,8 @@ ctl_refresh(void) } #ifdef JEMALLOC_STATS - ctl_stats.allocated = ctl_stats.arenas[narenas].astats.allocated_small - + ctl_stats.arenas[narenas].astats.allocated_medium + ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small + + ctl_stats.arenas[narenas].allocated_medium + ctl_stats.arenas[narenas].astats.allocated_large + ctl_stats.huge.allocated; ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT) @@ -1203,17 +1297,21 @@ CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t) CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t) CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t) CTL_RO_GEN(stats_arenas_i_small_allocated, - ctl_stats.arenas[mib[2]].astats.allocated_small, size_t) + ctl_stats.arenas[mib[2]].allocated_small, size_t) CTL_RO_GEN(stats_arenas_i_small_nmalloc, - ctl_stats.arenas[mib[2]].astats.nmalloc_small, uint64_t) + ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t) CTL_RO_GEN(stats_arenas_i_small_ndalloc, - ctl_stats.arenas[mib[2]].astats.ndalloc_small, uint64_t) + ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t) +CTL_RO_GEN(stats_arenas_i_small_nrequests, + ctl_stats.arenas[mib[2]].nrequests_small, uint64_t) CTL_RO_GEN(stats_arenas_i_medium_allocated, - ctl_stats.arenas[mib[2]].astats.allocated_medium, size_t) + ctl_stats.arenas[mib[2]].allocated_medium, size_t) CTL_RO_GEN(stats_arenas_i_medium_nmalloc, - ctl_stats.arenas[mib[2]].astats.nmalloc_medium, uint64_t) + ctl_stats.arenas[mib[2]].nmalloc_medium, uint64_t) CTL_RO_GEN(stats_arenas_i_medium_ndalloc, - ctl_stats.arenas[mib[2]].astats.ndalloc_medium, uint64_t) + ctl_stats.arenas[mib[2]].ndalloc_medium, uint64_t) +CTL_RO_GEN(stats_arenas_i_medium_nrequests, + ctl_stats.arenas[mib[2]].nrequests_medium, uint64_t) CTL_RO_GEN(stats_arenas_i_large_allocated, ctl_stats.arenas[mib[2]].astats.allocated_large, size_t) CTL_RO_GEN(stats_arenas_i_large_nmalloc, @@ -1221,6 +1319,12 @@ CTL_RO_GEN(stats_arenas_i_large_nmalloc, CTL_RO_GEN(stats_arenas_i_large_ndalloc, ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t) +CTL_RO_GEN(stats_arenas_i_bins_j_allocated, + ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t) +CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc, + ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t) +CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc, + ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t) CTL_RO_GEN(stats_arenas_i_bins_j_nrequests, ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t) #ifdef JEMALLOC_TCACHE diff --git a/jemalloc/src/stats.c b/jemalloc/src/stats.c index 236d7f87..dcd93f1c 100644 --- a/jemalloc/src/stats.c +++ b/jemalloc/src/stats.c @@ -160,12 +160,13 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("config.tcache", &config_tcache, bool); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs nrequests " - "nfills nflushes newruns reruns maxruns curruns\n"); + "bins: bin size regs pgs allocated nmalloc" + " ndalloc nrequests nfills nflushes newruns" + " reruns maxruns curruns\n"); } else { malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs nrequests " - "newruns reruns maxruns curruns\n"); + "bins: bin size regs pgs allocated nmalloc" + " ndalloc newruns reruns maxruns curruns\n"); } CTL_GET("arenas.nbins", &nbins, unsigned); for (j = 0, gap_start = UINT_MAX; j < nbins; j++) { @@ -177,9 +178,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, gap_start = j; } else { unsigned ntbins_, nqbins, ncbins, nsbins; - size_t reg_size, run_size; + size_t reg_size, run_size, allocated; uint32_t nregs; - uint64_t nrequests, nfills, nflushes, reruns; + uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; + uint64_t reruns; size_t highruns, curruns; if (gap_start != UINT_MAX) { @@ -202,9 +204,15 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_J_GET("arenas.bin.0.size", ®_size, size_t); CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t); CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t); - CTL_IJ_GET("stats.arenas.0.bins.0.nrequests", - &nrequests, uint64_t); + CTL_IJ_GET("stats.arenas.0.bins.0.allocated", + &allocated, size_t); + CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc", + &nmalloc, uint64_t); + CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc", + &ndalloc, uint64_t); if (config_tcache) { + CTL_IJ_GET("stats.arenas.0.bins.0.nrequests", + &nrequests, uint64_t); CTL_IJ_GET("stats.arenas.0.bins.0.nfills", &nfills, uint64_t); CTL_IJ_GET("stats.arenas.0.bins.0.nflushes", @@ -218,29 +226,32 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, size_t); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, - "%13u %1s %5zu %4u %3zu %10"PRIu64 - " %9"PRIu64" %9"PRIu64" %9"PRIu64"" - " %9"PRIu64" %7zu %7zu\n", + "%13u %1s %5zu %4u %3zu %9zu %9"PRIu64 + " %9"PRIu64" %10"PRIu64" %9"PRIu64 + " %9"PRIu64" %9"PRIu64" %9"PRIu64 + " %7zu %7zu\n", j, j < ntbins_ ? "T" : j < ntbins_ + nqbins ? "Q" : j < ntbins_ + nqbins + ncbins ? "C" : j < ntbins_ + nqbins + ncbins + nsbins ? "S" : "M", reg_size, nregs, run_size / pagesize, - nrequests, nfills, nflushes, nruns, reruns, - highruns, curruns); + allocated, nmalloc, ndalloc, nrequests, + nfills, nflushes, nruns, reruns, highruns, + curruns); } else { malloc_cprintf(write_cb, cbopaque, - "%13u %1s %5zu %4u %3zu %10"PRIu64 - " %9"PRIu64" %9"PRIu64" %7zu %7zu\n", + "%13u %1s %5zu %4u %3zu %9zu %9"PRIu64 + " %9"PRIu64" %9"PRIu64" %9"PRIu64 + " %7zu %7zu\n", j, j < ntbins_ ? "T" : j < ntbins_ + nqbins ? "Q" : j < ntbins_ + nqbins + ncbins ? "C" : j < ntbins_ + nqbins + ncbins + nsbins ? "S" : "M", reg_size, nregs, run_size / pagesize, - nrequests, nruns, reruns, highruns, - curruns); + allocated, nmalloc, ndalloc, nruns, reruns, + highruns, curruns); } } } @@ -305,9 +316,9 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, size_t pagesize, pactive, pdirty, mapped; uint64_t npurge, nmadvise, purged; size_t small_allocated; - uint64_t small_nmalloc, small_ndalloc; + uint64_t small_nmalloc, small_ndalloc, small_nrequests; size_t medium_allocated; - uint64_t medium_nmalloc, medium_ndalloc; + uint64_t medium_nmalloc, medium_ndalloc, medium_nrequests; size_t large_allocated; uint64_t large_nmalloc, large_ndalloc; @@ -325,30 +336,34 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, nmadvise, nmadvise == 1 ? "" : "s", purged); malloc_cprintf(write_cb, cbopaque, - " allocated nmalloc ndalloc\n"); + " allocated nmalloc ndalloc nrequests\n"); CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t); CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t); CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t); + CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t); malloc_cprintf(write_cb, cbopaque, - "small: %12zu %12"PRIu64" %12"PRIu64"\n", - small_allocated, small_nmalloc, small_ndalloc); + "small: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + small_allocated, small_nmalloc, small_ndalloc, small_nrequests); CTL_I_GET("stats.arenas.0.medium.allocated", &medium_allocated, size_t); CTL_I_GET("stats.arenas.0.medium.nmalloc", &medium_nmalloc, uint64_t); CTL_I_GET("stats.arenas.0.medium.ndalloc", &medium_ndalloc, uint64_t); + CTL_I_GET("stats.arenas.0.medium.nrequests", &medium_nrequests, + uint64_t); malloc_cprintf(write_cb, cbopaque, - "medium: %12zu %12"PRIu64" %12"PRIu64"\n", - medium_allocated, medium_nmalloc, medium_ndalloc); + "medium: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + medium_allocated, medium_nmalloc, medium_ndalloc, medium_nrequests); CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t); CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t); CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t); malloc_cprintf(write_cb, cbopaque, - "large: %12zu %12"PRIu64" %12"PRIu64"\n", - large_allocated, large_nmalloc, large_ndalloc); + "large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + large_allocated, large_nmalloc, large_ndalloc, large_nmalloc); malloc_cprintf(write_cb, cbopaque, - "total: %12zu %12"PRIu64" %12"PRIu64"\n", + "total: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", small_allocated + medium_allocated + large_allocated, small_nmalloc + medium_nmalloc + large_nmalloc, - small_ndalloc + medium_ndalloc + large_ndalloc); + small_ndalloc + medium_ndalloc + large_ndalloc, + small_nrequests + medium_nrequests + large_nmalloc); malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", pactive * pagesize ); CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); diff --git a/jemalloc/src/tcache.c b/jemalloc/src/tcache.c index dcb72c61..4192f98f 100644 --- a/jemalloc/src/tcache.c +++ b/jemalloc/src/tcache.c @@ -45,7 +45,7 @@ tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) void tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem -#ifdef JEMALLOC_PROF +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) , tcache_t *tcache #endif ) @@ -53,16 +53,29 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem void *flush, *deferred, *ptr; unsigned i, nflush, ndeferred; + assert(rem <= tbin->ncached); + for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL; flush = deferred, nflush = ndeferred) { - /* Lock the arena associated with the first object. */ + /* Lock the arena bin associated with the first object. */ arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush); arena_t *arena = chunk->arena; - malloc_mutex_lock(&arena->lock); -#ifdef JEMALLOC_PROF + arena_bin_t *bin = &arena->bins[binind]; + + malloc_mutex_lock(&bin->lock); +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) if (arena == tcache->arena) { +# ifdef JEMALLOC_PROF + malloc_mutex_lock(&arena->lock); arena_prof_accum(arena, tcache->prof_accumbytes); + malloc_mutex_unlock(&arena->lock); tcache->prof_accumbytes = 0; +# endif +# ifdef JEMALLOC_STATS + bin->stats.nflushes++; + bin->stats.nrequests += tbin->tstats.nrequests; + tbin->tstats.nrequests = 0; +# endif } #endif deferred = NULL; @@ -81,31 +94,16 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem } else { /* * This object was allocated via a different - * arena than the one that is currently locked. - * Stash the object, so that it can be handled - * in a future pass. + * arena bin than the one that is currently + * locked. Stash the object, so that it can be + * handled in a future pass. */ *(void **)ptr = deferred; deferred = ptr; ndeferred++; } } -#ifdef JEMALLOC_STATS - arena->bins[binind].stats.nflushes++; - { - arena_bin_t *bin = &arena->bins[binind]; - bin->stats.nrequests += tbin->tstats.nrequests; - if (bin->reg_size <= small_maxclass) { - arena->stats.nmalloc_small += - tbin->tstats.nrequests; - } else { - arena->stats.nmalloc_medium += - tbin->tstats.nrequests; - } - tbin->tstats.nrequests = 0; - } -#endif - malloc_mutex_unlock(&arena->lock); + malloc_mutex_unlock(&bin->lock); if (flush != NULL) { /* @@ -117,6 +115,8 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem } tbin->ncached = rem; + if (tbin->ncached < tbin->low_water) + tbin->low_water = tbin->ncached; } tcache_t * @@ -178,14 +178,14 @@ tcache_destroy(tcache_t *tcache) /* Unlink from list of extant tcaches. */ malloc_mutex_lock(&tcache->arena->lock); ql_remove(&tcache->arena->tcache_ql, tcache, link); - tcache_stats_merge(tcache, tcache->arena); malloc_mutex_unlock(&tcache->arena->lock); + tcache_stats_merge(tcache, tcache->arena); #endif for (i = 0; i < nbins; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; tcache_bin_flush(tbin, i, 0 -#ifdef JEMALLOC_PROF +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) , tcache #endif ); @@ -194,16 +194,9 @@ tcache_destroy(tcache_t *tcache) if (tbin->tstats.nrequests != 0) { arena_t *arena = tcache->arena; arena_bin_t *bin = &arena->bins[i]; - malloc_mutex_lock(&arena->lock); + malloc_mutex_lock(&bin->lock); bin->stats.nrequests += tbin->tstats.nrequests; - if (bin->reg_size <= small_maxclass) { - arena->stats.nmalloc_small += - tbin->tstats.nrequests; - } else { - arena->stats.nmalloc_medium += - tbin->tstats.nrequests; - } - malloc_mutex_unlock(&arena->lock); + malloc_mutex_unlock(&bin->lock); } #endif } @@ -222,10 +215,14 @@ tcache_destroy(tcache_t *tcache) size_t pageind = (((uintptr_t)tcache - (uintptr_t)chunk) >> PAGE_SHIFT); arena_chunk_map_t *mapelm = &chunk->map[pageind]; + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)((pageind - ((mapelm->bits & CHUNK_MAP_PG_MASK) + >> CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT)); + arena_bin_t *bin = run->bin; - malloc_mutex_lock(&arena->lock); + malloc_mutex_lock(&bin->lock); arena_dalloc_bin(arena, chunk, tcache, mapelm); - malloc_mutex_unlock(&arena->lock); + malloc_mutex_unlock(&bin->lock); } else idalloc(tcache); } @@ -250,18 +247,12 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena) unsigned i; /* Merge and reset tcache stats. */ - for (i = 0; i < mbin0; i++) { + for (i = 0; i < nbins; i++) { arena_bin_t *bin = &arena->bins[i]; tcache_bin_t *tbin = &tcache->tbins[i]; + malloc_mutex_lock(&bin->lock); bin->stats.nrequests += tbin->tstats.nrequests; - arena->stats.nmalloc_small += tbin->tstats.nrequests; - tbin->tstats.nrequests = 0; - } - for (; i < nbins; i++) { - arena_bin_t *bin = &arena->bins[i]; - tcache_bin_t *tbin = &tcache->tbins[i]; - bin->stats.nrequests += tbin->tstats.nrequests; - arena->stats.nmalloc_medium += tbin->tstats.nrequests; + malloc_mutex_unlock(&bin->lock); tbin->tstats.nrequests = 0; } }