Refactor ctl_stats_t.
Refactor ctl_stats_t to be a demand-zeroed non-growing data structure. To keep the size from being onerous (~60 MiB) on 32-bit systems, convert the arenas field to contain pointers rather than directly embedded ctl_arena_stats_t elements.
This commit is contained in:
Jason Evans
parent
0f04bb1d6f
commit
d778dd2afc
@ -61,7 +61,7 @@ struct ctl_stats_s {
|
||||
size_t mapped;
|
||||
size_t retained;
|
||||
unsigned narenas;
|
||||
ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */
|
||||
ctl_arena_stats_t *arenas[1 << MALLOCX_ARENA_BITS];
|
||||
};
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
|
||||
@ -208,11 +208,18 @@ typedef unsigned szind_t;
|
||||
*
|
||||
* aaaaaaaa aaaatttt tttttttt 0znnnnnn
|
||||
*/
|
||||
#define MALLOCX_ARENA_MASK ((int)~0xfffff)
|
||||
#define MALLOCX_ARENA_MAX 0xffe
|
||||
#define MALLOCX_TCACHE_MASK ((int)~0xfff000ffU)
|
||||
#define MALLOCX_TCACHE_MAX 0xffd
|
||||
#define MALLOCX_LG_ALIGN_MASK ((int)0x3f)
|
||||
#define MALLOCX_ARENA_BITS 12
|
||||
#define MALLOCX_TCACHE_BITS 12
|
||||
#define MALLOCX_LG_ALIGN_BITS 6
|
||||
#define MALLOCX_ARENA_SHIFT 20
|
||||
#define MALLOCX_TCACHE_SHIFT 8
|
||||
#define MALLOCX_ARENA_MASK \
|
||||
(((1 << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT)
|
||||
#define MALLOCX_ARENA_MAX ((1 << MALLOCX_ARENA_BITS) - 2)
|
||||
#define MALLOCX_TCACHE_MASK \
|
||||
(((1 << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT)
|
||||
#define MALLOCX_TCACHE_MAX ((1 << MALLOCX_TCACHE_BITS) - 3)
|
||||
#define MALLOCX_LG_ALIGN_MASK ((1 << MALLOCX_LG_ALIGN_BITS) - 1)
|
||||
/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */
|
||||
#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \
|
||||
(ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
|
||||
@ -222,9 +229,9 @@ typedef unsigned szind_t;
|
||||
((bool)(flags & MALLOCX_ZERO))
|
||||
|
||||
#define MALLOCX_TCACHE_GET(flags) \
|
||||
(((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2)
|
||||
(((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) - 2)
|
||||
#define MALLOCX_ARENA_GET(flags) \
|
||||
(((unsigned)(((unsigned)flags) >> 20)) - 1)
|
||||
(((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1)
|
||||
|
||||
/* Smallest size class to support. */
|
||||
#define TINY_MIN (1U << LG_TINY_MIN)
|
||||
|
||||
151
src/ctl.c
151
src/ctl.c
@ -6,12 +6,12 @@
|
||||
|
||||
/*
|
||||
* ctl_mtx protects the following:
|
||||
* - ctl_stats.*
|
||||
* - ctl_stats->*
|
||||
*/
|
||||
static malloc_mutex_t ctl_mtx;
|
||||
static bool ctl_initialized;
|
||||
static uint64_t ctl_epoch;
|
||||
static ctl_stats_t ctl_stats;
|
||||
static ctl_stats_t *ctl_stats;
|
||||
|
||||
/******************************************************************************/
|
||||
/* Helpers for named and indexed nodes. */
|
||||
@ -455,7 +455,7 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
|
||||
a = 0;
|
||||
break;
|
||||
default:
|
||||
if (compat && i == ctl_stats.narenas) {
|
||||
if (compat && i == ctl_stats->narenas) {
|
||||
/*
|
||||
* Provide deprecated backward compatibility for
|
||||
* accessing the merged stats at index narenas rather
|
||||
@ -463,7 +463,7 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
|
||||
* removal in 6.0.0.
|
||||
*/
|
||||
a = 0;
|
||||
} else if (validate && i >= ctl_stats.narenas)
|
||||
} else if (validate && i >= ctl_stats->narenas)
|
||||
a = UINT_MAX;
|
||||
else {
|
||||
/*
|
||||
@ -471,8 +471,8 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
|
||||
* more than one past the range of indices that have
|
||||
* initialized stats.
|
||||
*/
|
||||
assert(i < ctl_stats.narenas || (!validate && i ==
|
||||
ctl_stats.narenas));
|
||||
assert(i < ctl_stats->narenas || (!validate && i ==
|
||||
ctl_stats->narenas));
|
||||
a = (unsigned)i + 1;
|
||||
}
|
||||
break;
|
||||
@ -481,11 +481,32 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
|
||||
return (a);
|
||||
}
|
||||
|
||||
static ctl_arena_stats_t *
|
||||
stats_arenas_i_impl(tsdn_t *tsdn, size_t i, bool compat, bool init)
|
||||
{
|
||||
ctl_arena_stats_t *ret;
|
||||
|
||||
assert(!compat || !init);
|
||||
|
||||
ret = ctl_stats->arenas[stats_arenas_i2a_impl(i, compat, false)];
|
||||
if (init && ret == NULL) {
|
||||
ret = (ctl_arena_stats_t *)base_alloc(tsdn, b0get(),
|
||||
sizeof(ctl_arena_stats_t), QUANTUM);
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
ctl_stats->arenas[stats_arenas_i2a_impl(i, compat, false)] =
|
||||
ret;
|
||||
}
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static ctl_arena_stats_t *
|
||||
stats_arenas_i(size_t i)
|
||||
{
|
||||
|
||||
return (&ctl_stats.arenas[stats_arenas_i2a_impl(i, true, false)]);
|
||||
ctl_arena_stats_t *ret = stats_arenas_i_impl(TSDN_NULL, i, true, false);
|
||||
assert(ret != NULL);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -609,25 +630,15 @@ ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i)
|
||||
static bool
|
||||
ctl_grow(tsdn_t *tsdn, extent_hooks_t *extent_hooks)
|
||||
{
|
||||
ctl_arena_stats_t *astats;
|
||||
|
||||
/* Trigger stats allocation. */
|
||||
if (stats_arenas_i_impl(tsdn, ctl_stats->narenas, false, true) == NULL)
|
||||
return (true);
|
||||
|
||||
/* Initialize new arena. */
|
||||
if (arena_init(tsdn, ctl_stats.narenas, extent_hooks) == NULL)
|
||||
if (arena_init(tsdn, ctl_stats->narenas, extent_hooks) == NULL)
|
||||
return (true);
|
||||
|
||||
/* Allocate extended arena stats. */
|
||||
astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) *
|
||||
sizeof(ctl_arena_stats_t));
|
||||
if (astats == NULL)
|
||||
return (true);
|
||||
|
||||
/* Initialize the new astats element. */
|
||||
memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *
|
||||
sizeof(ctl_arena_stats_t));
|
||||
memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
|
||||
a0dalloc(ctl_stats.arenas);
|
||||
ctl_stats.arenas = astats;
|
||||
ctl_stats.narenas++;
|
||||
ctl_stats->narenas++;
|
||||
|
||||
return (false);
|
||||
}
|
||||
@ -637,7 +648,7 @@ ctl_refresh(tsdn_t *tsdn)
|
||||
{
|
||||
unsigned i;
|
||||
ctl_arena_stats_t *sstats = stats_arenas_i(MALLCTL_ARENAS_ALL);
|
||||
VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);
|
||||
VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats->narenas);
|
||||
|
||||
/*
|
||||
* Clear sum stats, since they will be merged into by
|
||||
@ -645,10 +656,10 @@ ctl_refresh(tsdn_t *tsdn)
|
||||
*/
|
||||
ctl_arena_clear(sstats);
|
||||
|
||||
for (i = 0; i < ctl_stats.narenas; i++)
|
||||
for (i = 0; i < ctl_stats->narenas; i++)
|
||||
tarenas[i] = arena_get(tsdn, i, false);
|
||||
|
||||
for (i = 0; i < ctl_stats.narenas; i++) {
|
||||
for (i = 0; i < ctl_stats->narenas; i++) {
|
||||
ctl_arena_stats_t *astats = stats_arenas_i(i);
|
||||
bool initialized = (tarenas[i] != NULL);
|
||||
|
||||
@ -658,14 +669,14 @@ ctl_refresh(tsdn_t *tsdn)
|
||||
}
|
||||
|
||||
if (config_stats) {
|
||||
ctl_stats.allocated = sstats->allocated_small +
|
||||
ctl_stats->allocated = sstats->allocated_small +
|
||||
sstats->astats.allocated_large;
|
||||
ctl_stats.active = (sstats->pactive << LG_PAGE);
|
||||
ctl_stats.metadata = sstats->astats.base +
|
||||
ctl_stats->active = (sstats->pactive << LG_PAGE);
|
||||
ctl_stats->metadata = sstats->astats.base +
|
||||
sstats->astats.internal;
|
||||
ctl_stats.resident = sstats->astats.resident;
|
||||
ctl_stats.mapped = sstats->astats.mapped;
|
||||
ctl_stats.retained = sstats->astats.retained;
|
||||
ctl_stats->resident = sstats->astats.resident;
|
||||
ctl_stats->mapped = sstats->astats.mapped;
|
||||
ctl_stats->retained = sstats->astats.retained;
|
||||
}
|
||||
|
||||
ctl_epoch++;
|
||||
@ -678,20 +689,41 @@ ctl_init(tsdn_t *tsdn)
|
||||
|
||||
malloc_mutex_lock(tsdn, &ctl_mtx);
|
||||
if (!ctl_initialized) {
|
||||
ctl_arena_stats_t *sstats;
|
||||
unsigned i;
|
||||
|
||||
/*
|
||||
* Allocate space for one extra arena stats element, which
|
||||
* contains summed stats across all arenas.
|
||||
* Allocate demand-zeroed space for pointers to the full range
|
||||
* of supported arena indices.
|
||||
*/
|
||||
ctl_stats.narenas = narenas_total_get();
|
||||
ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc(
|
||||
(ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t));
|
||||
if (ctl_stats.arenas == NULL) {
|
||||
if (ctl_stats == NULL) {
|
||||
ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(),
|
||||
sizeof(ctl_stats_t), QUANTUM);
|
||||
if (ctl_stats == NULL) {
|
||||
ret = true;
|
||||
goto label_return;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate space for the current full range of arenas here
|
||||
* rather than doing it lazily elsewhere, in order to limit when
|
||||
* OOM-caused errors can occur.
|
||||
*/
|
||||
if ((sstats = stats_arenas_i_impl(tsdn, MALLCTL_ARENAS_ALL,
|
||||
false, true)) == NULL) {
|
||||
ret = true;
|
||||
goto label_return;
|
||||
}
|
||||
memset(ctl_stats.arenas, 0, (ctl_stats.narenas + 1) *
|
||||
sizeof(ctl_arena_stats_t));
|
||||
stats_arenas_i(MALLCTL_ARENAS_ALL)->initialized = true;
|
||||
sstats->initialized = true;
|
||||
|
||||
ctl_stats->narenas = narenas_total_get();
|
||||
for (i = 0; i < ctl_stats->narenas; i++) {
|
||||
if (stats_arenas_i_impl(tsdn, i, false, true) == NULL) {
|
||||
ret = true;
|
||||
goto label_return;
|
||||
}
|
||||
}
|
||||
|
||||
ctl_epoch = 0;
|
||||
ctl_refresh(tsdn);
|
||||
@ -1428,7 +1460,7 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all)
|
||||
|
||||
malloc_mutex_lock(tsdn, &ctl_mtx);
|
||||
{
|
||||
unsigned narenas = ctl_stats.narenas;
|
||||
unsigned narenas = ctl_stats->narenas;
|
||||
|
||||
/*
|
||||
* Access via index narenas is deprecated, and scheduled for
|
||||
@ -1511,7 +1543,7 @@ arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
|
||||
arena_ind = (unsigned)mib[1];
|
||||
if (config_debug) {
|
||||
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
|
||||
assert(arena_ind < ctl_stats.narenas);
|
||||
assert(arena_ind < ctl_stats->narenas);
|
||||
malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
|
||||
}
|
||||
assert(arena_ind >= opt_narenas);
|
||||
@ -1563,7 +1595,8 @@ arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
|
||||
* Access via index narenas is deprecated, and scheduled for removal in
|
||||
* 6.0.0.
|
||||
*/
|
||||
if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == ctl_stats.narenas) {
|
||||
if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind ==
|
||||
ctl_stats->narenas) {
|
||||
if (dss_prec != dss_prec_limit &&
|
||||
extent_dss_prec_set(dss_prec)) {
|
||||
ret = EFAULT;
|
||||
@ -1664,7 +1697,7 @@ arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
|
||||
const ctl_named_node_t *ret;
|
||||
|
||||
malloc_mutex_lock(tsdn, &ctl_mtx);
|
||||
if (i > ctl_stats.narenas && i != MALLCTL_ARENAS_ALL) {
|
||||
if (i > ctl_stats->narenas && i != MALLCTL_ARENAS_ALL) {
|
||||
ret = NULL;
|
||||
goto label_return;
|
||||
}
|
||||
@ -1690,7 +1723,7 @@ arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
|
||||
ret = EINVAL;
|
||||
goto label_return;
|
||||
}
|
||||
narenas = ctl_stats.narenas;
|
||||
narenas = ctl_stats->narenas;
|
||||
READ(narenas, unsigned);
|
||||
|
||||
ret = 0;
|
||||
@ -1708,13 +1741,13 @@ arenas_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
|
||||
|
||||
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
|
||||
READONLY();
|
||||
if (*oldlenp != ctl_stats.narenas * sizeof(bool)) {
|
||||
if (*oldlenp != ctl_stats->narenas * sizeof(bool)) {
|
||||
ret = EINVAL;
|
||||
nread = (*oldlenp < ctl_stats.narenas * sizeof(bool))
|
||||
? (unsigned)(*oldlenp / sizeof(bool)) : ctl_stats.narenas;
|
||||
nread = (*oldlenp < ctl_stats->narenas * sizeof(bool))
|
||||
? (unsigned)(*oldlenp / sizeof(bool)) : ctl_stats->narenas;
|
||||
} else {
|
||||
ret = 0;
|
||||
nread = ctl_stats.narenas;
|
||||
nread = ctl_stats->narenas;
|
||||
}
|
||||
|
||||
for (i = 0; i < nread; i++)
|
||||
@ -1795,7 +1828,7 @@ arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
|
||||
ret = EAGAIN;
|
||||
goto label_return;
|
||||
}
|
||||
narenas = ctl_stats.narenas - 1;
|
||||
narenas = ctl_stats->narenas - 1;
|
||||
READ(narenas, unsigned);
|
||||
|
||||
ret = 0;
|
||||
@ -1932,12 +1965,12 @@ CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t)
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats.metadata, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_resident, ctl_stats.resident, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_retained, ctl_stats.retained, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
|
||||
CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
|
||||
|
||||
CTL_RO_GEN(stats_arenas_i_dss, stats_arenas_i(mib[2])->dss, const char *)
|
||||
CTL_RO_GEN(stats_arenas_i_decay_time, stats_arenas_i(mib[2])->decay_time,
|
||||
@ -2036,7 +2069,7 @@ stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
|
||||
|
||||
malloc_mutex_lock(tsdn, &ctl_mtx);
|
||||
a = stats_arenas_i2a_impl(i, true, true);
|
||||
if (a == UINT_MAX || !ctl_stats.arenas[a].initialized) {
|
||||
if (a == UINT_MAX || !ctl_stats->arenas[a]->initialized) {
|
||||
ret = NULL;
|
||||
goto label_return;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user