project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor ctl_stats_t.

Browse Source 
Refactor ctl_stats_t to be a demand-zeroed non-growing data structure.
To keep the size from being onerous (~60 MiB) on 32-bit systems, convert
the arenas field to contain pointers rather than directly embedded
ctl_arena_stats_t elements.
This commit is contained in:
Jason Evans 2017-01-03 12:40:54 -08:00
parent 0f04bb1d6f
commit d778dd2afc
3 changed files with 107 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/jemalloc/internal/ctl.h
View File

@ -61,7 +61,7 @@ struct ctl_stats_s {
size_t mapped; size_t mapped;
size_t retained; size_t retained;
unsigned narenas; unsigned narenas;
ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ ctl_arena_stats_t *arenas[1 << MALLOCX_ARENA_BITS];
}; };
#endif /* JEMALLOC_H_STRUCTS */ #endif /* JEMALLOC_H_STRUCTS */

21
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -208,11 +208,18 @@ typedef unsigned szind_t;
* *
* aaaaaaaa aaaatttt tttttttt 0znnnnnn * aaaaaaaa aaaatttt tttttttt 0znnnnnn
*/ */
#define MALLOCX_ARENA_MASK ((int)~0xfffff) #define MALLOCX_ARENA_BITS 12
#define MALLOCX_ARENA_MAX 0xffe #define MALLOCX_TCACHE_BITS 12
#define MALLOCX_TCACHE_MASK ((int)~0xfff000ffU) #define MALLOCX_LG_ALIGN_BITS 6
#define MALLOCX_TCACHE_MAX 0xffd #define MALLOCX_ARENA_SHIFT 20
#define MALLOCX_LG_ALIGN_MASK ((int)0x3f) #define MALLOCX_TCACHE_SHIFT 8
#define MALLOCX_ARENA_MASK \
(((1 << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT)
#define MALLOCX_ARENA_MAX ((1 << MALLOCX_ARENA_BITS) - 2)
#define MALLOCX_TCACHE_MASK \
(((1 << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT)
#define MALLOCX_TCACHE_MAX ((1 << MALLOCX_TCACHE_BITS) - 3)
#define MALLOCX_LG_ALIGN_MASK ((1 << MALLOCX_LG_ALIGN_BITS) - 1)
/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */ /* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */
#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ #define MALLOCX_ALIGN_GET_SPECIFIED(flags) \
(ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)) (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
@ -222,9 +229,9 @@ typedef unsigned szind_t;
((bool)(flags & MALLOCX_ZERO)) ((bool)(flags & MALLOCX_ZERO))
#define MALLOCX_TCACHE_GET(flags) \ #define MALLOCX_TCACHE_GET(flags) \
(((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2) (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) - 2)
#define MALLOCX_ARENA_GET(flags) \ #define MALLOCX_ARENA_GET(flags) \
(((unsigned)(((unsigned)flags) >> 20)) - 1) (((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1)
/* Smallest size class to support. */ /* Smallest size class to support. */
#define TINY_MIN (1U << LG_TINY_MIN) #define TINY_MIN (1U << LG_TINY_MIN)

151
src/ctl.c
View File

@ -6,12 +6,12 @@
/* /*
* ctl_mtx protects the following: * ctl_mtx protects the following:
* - ctl_stats.* * - ctl_stats->*
*/ */
static malloc_mutex_t ctl_mtx; static malloc_mutex_t ctl_mtx;
static bool ctl_initialized; static bool ctl_initialized;
static uint64_t ctl_epoch; static uint64_t ctl_epoch;
static ctl_stats_t ctl_stats; static ctl_stats_t *ctl_stats;
/******************************************************************************/ /******************************************************************************/
/* Helpers for named and indexed nodes. */ /* Helpers for named and indexed nodes. */
@ -455,7 +455,7 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
a = 0; a = 0;
break; break;
default: default:
if (compat && i == ctl_stats.narenas) { if (compat && i == ctl_stats->narenas) {
/* /*
* Provide deprecated backward compatibility for * Provide deprecated backward compatibility for
* accessing the merged stats at index narenas rather * accessing the merged stats at index narenas rather
@ -463,7 +463,7 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
* removal in 6.0.0. * removal in 6.0.0.
*/ */
a = 0; a = 0;
} else if (validate && i >= ctl_stats.narenas) } else if (validate && i >= ctl_stats->narenas)
a = UINT_MAX; a = UINT_MAX;
else { else {
/* /*
@ -471,8 +471,8 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
* more than one past the range of indices that have * more than one past the range of indices that have
* initialized stats. * initialized stats.
*/ */
assert(i < ctl_stats.narenas || (!validate && i == assert(i < ctl_stats->narenas || (!validate && i ==
ctl_stats.narenas)); ctl_stats->narenas));
a = (unsigned)i + 1; a = (unsigned)i + 1;
} }
break; break;
@ -481,11 +481,32 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
return (a); return (a);
} }
static ctl_arena_stats_t *
stats_arenas_i_impl(tsdn_t *tsdn, size_t i, bool compat, bool init)
{
ctl_arena_stats_t *ret;
assert(!compat || !init);
ret = ctl_stats->arenas[stats_arenas_i2a_impl(i, compat, false)];
if (init && ret == NULL) {
ret = (ctl_arena_stats_t *)base_alloc(tsdn, b0get(),
sizeof(ctl_arena_stats_t), QUANTUM);
if (ret == NULL)
return (NULL);
ctl_stats->arenas[stats_arenas_i2a_impl(i, compat, false)] =
ret;
}
return (ret);
}
static ctl_arena_stats_t * static ctl_arena_stats_t *
stats_arenas_i(size_t i) stats_arenas_i(size_t i)
{ {
ctl_arena_stats_t *ret = stats_arenas_i_impl(TSDN_NULL, i, true, false);
return (&ctl_stats.arenas[stats_arenas_i2a_impl(i, true, false)]); assert(ret != NULL);
return (ret);
} }
static void static void
@ -609,25 +630,15 @@ ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i)
static bool static bool
ctl_grow(tsdn_t *tsdn, extent_hooks_t *extent_hooks) ctl_grow(tsdn_t *tsdn, extent_hooks_t *extent_hooks)
{ {
ctl_arena_stats_t *astats;
/* Trigger stats allocation. */
if (stats_arenas_i_impl(tsdn, ctl_stats->narenas, false, true) == NULL)
return (true);
/* Initialize new arena. */ /* Initialize new arena. */
if (arena_init(tsdn, ctl_stats.narenas, extent_hooks) == NULL) if (arena_init(tsdn, ctl_stats->narenas, extent_hooks) == NULL)
return (true); return (true);
ctl_stats->narenas++;
/* Allocate extended arena stats. */
astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) *
sizeof(ctl_arena_stats_t));
if (astats == NULL)
return (true);
/* Initialize the new astats element. */
memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *
sizeof(ctl_arena_stats_t));
memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
a0dalloc(ctl_stats.arenas);
ctl_stats.arenas = astats;
ctl_stats.narenas++;
return (false); return (false);
} }
@ -637,7 +648,7 @@ ctl_refresh(tsdn_t *tsdn)
{ {
unsigned i; unsigned i;
ctl_arena_stats_t *sstats = stats_arenas_i(MALLCTL_ARENAS_ALL); ctl_arena_stats_t *sstats = stats_arenas_i(MALLCTL_ARENAS_ALL);
VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats->narenas);
/* /*
* Clear sum stats, since they will be merged into by * Clear sum stats, since they will be merged into by
@ -645,10 +656,10 @@ ctl_refresh(tsdn_t *tsdn)
*/ */
ctl_arena_clear(sstats); ctl_arena_clear(sstats);
for (i = 0; i < ctl_stats.narenas; i++) for (i = 0; i < ctl_stats->narenas; i++)
tarenas[i] = arena_get(tsdn, i, false); tarenas[i] = arena_get(tsdn, i, false);
for (i = 0; i < ctl_stats.narenas; i++) { for (i = 0; i < ctl_stats->narenas; i++) {
ctl_arena_stats_t *astats = stats_arenas_i(i); ctl_arena_stats_t *astats = stats_arenas_i(i);
bool initialized = (tarenas[i] != NULL); bool initialized = (tarenas[i] != NULL);
@ -658,14 +669,14 @@ ctl_refresh(tsdn_t *tsdn)
} }
if (config_stats) { if (config_stats) {
ctl_stats.allocated = sstats->allocated_small + ctl_stats->allocated = sstats->allocated_small +
sstats->astats.allocated_large; sstats->astats.allocated_large;
ctl_stats.active = (sstats->pactive << LG_PAGE); ctl_stats->active = (sstats->pactive << LG_PAGE);
ctl_stats.metadata = sstats->astats.base + ctl_stats->metadata = sstats->astats.base +
sstats->astats.internal; sstats->astats.internal;
ctl_stats.resident = sstats->astats.resident; ctl_stats->resident = sstats->astats.resident;
ctl_stats.mapped = sstats->astats.mapped; ctl_stats->mapped = sstats->astats.mapped;
ctl_stats.retained = sstats->astats.retained; ctl_stats->retained = sstats->astats.retained;
} }
ctl_epoch++; ctl_epoch++;
@ -678,20 +689,41 @@ ctl_init(tsdn_t *tsdn)
malloc_mutex_lock(tsdn, &ctl_mtx); malloc_mutex_lock(tsdn, &ctl_mtx);
if (!ctl_initialized) { if (!ctl_initialized) {
ctl_arena_stats_t *sstats;
unsigned i;
/* /*
* Allocate space for one extra arena stats element, which * Allocate demand-zeroed space for pointers to the full range
* contains summed stats across all arenas. * of supported arena indices.
*/ */
ctl_stats.narenas = narenas_total_get(); if (ctl_stats == NULL) {
ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc( ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(),
(ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); sizeof(ctl_stats_t), QUANTUM);
if (ctl_stats.arenas == NULL) { if (ctl_stats == NULL) {
ret = true; ret = true;
goto label_return; goto label_return;
} }
memset(ctl_stats.arenas, 0, (ctl_stats.narenas + 1) * }
sizeof(ctl_arena_stats_t));
stats_arenas_i(MALLCTL_ARENAS_ALL)->initialized = true; /*
* Allocate space for the current full range of arenas here
* rather than doing it lazily elsewhere, in order to limit when
* OOM-caused errors can occur.
*/
if ((sstats = stats_arenas_i_impl(tsdn, MALLCTL_ARENAS_ALL,
false, true)) == NULL) {
ret = true;
goto label_return;
}
sstats->initialized = true;
ctl_stats->narenas = narenas_total_get();
for (i = 0; i < ctl_stats->narenas; i++) {
if (stats_arenas_i_impl(tsdn, i, false, true) == NULL) {
ret = true;
goto label_return;
}
}
ctl_epoch = 0; ctl_epoch = 0;
ctl_refresh(tsdn); ctl_refresh(tsdn);
@ -1428,7 +1460,7 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all)
malloc_mutex_lock(tsdn, &ctl_mtx); malloc_mutex_lock(tsdn, &ctl_mtx);
{ {
unsigned narenas = ctl_stats.narenas; unsigned narenas = ctl_stats->narenas;
/* /*
* Access via index narenas is deprecated, and scheduled for * Access via index narenas is deprecated, and scheduled for
@ -1511,7 +1543,7 @@ arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
arena_ind = (unsigned)mib[1]; arena_ind = (unsigned)mib[1];
if (config_debug) { if (config_debug) {
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
assert(arena_ind < ctl_stats.narenas); assert(arena_ind < ctl_stats->narenas);
malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
} }
assert(arena_ind >= opt_narenas); assert(arena_ind >= opt_narenas);
@ -1563,7 +1595,8 @@ arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
* Access via index narenas is deprecated, and scheduled for removal in * Access via index narenas is deprecated, and scheduled for removal in
* 6.0.0. * 6.0.0.
*/ */
if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == ctl_stats.narenas) { if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind ==
ctl_stats->narenas) {
if (dss_prec != dss_prec_limit && if (dss_prec != dss_prec_limit &&
extent_dss_prec_set(dss_prec)) { extent_dss_prec_set(dss_prec)) {
ret = EFAULT; ret = EFAULT;
@ -1664,7 +1697,7 @@ arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
const ctl_named_node_t *ret; const ctl_named_node_t *ret;
malloc_mutex_lock(tsdn, &ctl_mtx); malloc_mutex_lock(tsdn, &ctl_mtx);
if (i > ctl_stats.narenas && i != MALLCTL_ARENAS_ALL) { if (i > ctl_stats->narenas && i != MALLCTL_ARENAS_ALL) {
ret = NULL; ret = NULL;
goto label_return; goto label_return;
} }
@ -1690,7 +1723,7 @@ arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = EINVAL; ret = EINVAL;
goto label_return; goto label_return;
} }
narenas = ctl_stats.narenas; narenas = ctl_stats->narenas;
READ(narenas, unsigned); READ(narenas, unsigned);
ret = 0; ret = 0;
@ -1708,13 +1741,13 @@ arenas_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
READONLY(); READONLY();
if (*oldlenp != ctl_stats.narenas * sizeof(bool)) { if (*oldlenp != ctl_stats->narenas * sizeof(bool)) {
ret = EINVAL; ret = EINVAL;
nread = (*oldlenp < ctl_stats.narenas * sizeof(bool)) nread = (*oldlenp < ctl_stats->narenas * sizeof(bool))
? (unsigned)(*oldlenp / sizeof(bool)) : ctl_stats.narenas; ? (unsigned)(*oldlenp / sizeof(bool)) : ctl_stats->narenas;
} else { } else {
ret = 0; ret = 0;
nread = ctl_stats.narenas; nread = ctl_stats->narenas;
} }
for (i = 0; i < nread; i++) for (i = 0; i < nread; i++)
@ -1795,7 +1828,7 @@ arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = EAGAIN; ret = EAGAIN;
goto label_return; goto label_return;
} }
narenas = ctl_stats.narenas - 1; narenas = ctl_stats->narenas - 1;
READ(narenas, unsigned); READ(narenas, unsigned);
ret = 0; ret = 0;
@ -1932,12 +1965,12 @@ CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t)
/******************************************************************************/ /******************************************************************************/
CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t)
CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t)
CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats.metadata, size_t) CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t)
CTL_RO_CGEN(config_stats, stats_resident, ctl_stats.resident, size_t) CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t)
CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
CTL_RO_CGEN(config_stats, stats_retained, ctl_stats.retained, size_t) CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
CTL_RO_GEN(stats_arenas_i_dss, stats_arenas_i(mib[2])->dss, const char *) CTL_RO_GEN(stats_arenas_i_dss, stats_arenas_i(mib[2])->dss, const char *)
CTL_RO_GEN(stats_arenas_i_decay_time, stats_arenas_i(mib[2])->decay_time, CTL_RO_GEN(stats_arenas_i_decay_time, stats_arenas_i(mib[2])->decay_time,
@ -2036,7 +2069,7 @@ stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
malloc_mutex_lock(tsdn, &ctl_mtx); malloc_mutex_lock(tsdn, &ctl_mtx);
a = stats_arenas_i2a_impl(i, true, true); a = stats_arenas_i2a_impl(i, true, true);
if (a == UINT_MAX || !ctl_stats.arenas[a].initialized) { if (a == UINT_MAX || !ctl_stats->arenas[a]->initialized) {
ret = NULL; ret = NULL;
goto label_return; goto label_return;
} }