diff --git a/Makefile.in b/Makefile.in
index d8704923..edc50b4b 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -195,6 +195,10 @@ TESTS_UNIT := \
$(srcroot)test/unit/util.c \
$(srcroot)test/unit/witness.c \
$(srcroot)test/unit/zero.c
+ifeq (@enable_prof@, 1)
+TESTS_UNIT += \
+ $(srcroot)test/unit/arena_reset_prof.c
+endif
TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
$(srcroot)test/integration/allocated.c \
$(srcroot)test/integration/extent.c \
diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index f213a2c8..36ec140b 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -426,13 +426,14 @@ for (i = 0; i < nbins; i++) {
mallctl*() functions internally, so inconsistent
statistics can be reported if multiple threads use these functions
simultaneously. If is specified during
- configuration, m
and a
can be specified to
- omit merged arena and per arena statistics, respectively; b
- and l
can be specified to omit per size class statistics
- for bins and large objects, respectively. Unrecognized characters are
- silently ignored. Note that thread caching may prevent some statistics
- from being completely up to date, since extra locking would be required to
- merge counters that track thread cache operations.
+ configuration, m
, d
, and a
+ can be specified to omit merged arena, destroyed merged arena, and per
+ arena statistics, respectively; b
and l
can
+ be specified to omit per size class statistics for bins and large objects,
+ respectively. Unrecognized characters are silently ignored. Note that
+ thread caching may prevent some statistics from being completely up to
+ date, since extra locking would be required to merge counters that track
+ thread cache operations.
The malloc_usable_size() function
returns the usable size of the allocation pointed to by
@@ -687,18 +688,21 @@ for (i = 0; i < nbins; i++) {
MALLCTL NAMESPACE
The following names are defined in the namespace accessible via the
- mallctl*() functions. Value types are
- specified in parentheses, their readable/writable statuses are encoded as
+ mallctl*() functions. Value types are specified in
+ parentheses, their readable/writable statuses are encoded as
rw, r-, -w, or
--, and required build configuration flags follow, if
any. A name element encoded as <i> or
<j> indicates an integer component, where the
integer varies from 0 to some upper value that must be determined via
introspection. In the case of stats.arenas.<i>.*
- and arena.<i>.{purge,decay,dss},
+ and arena.<i>.{initialized,purge,decay,dss},
<i> equal to
MALLCTL_ARENAS_ALL can be used to operate on all arenas
- or access the summation of statistics from all arenas. This constant can be
+ or access the summation of statistics from all arenas; similarly
+ <i> equal to
+ MALLCTL_ARENAS_DESTROYED can be used to access the
+ summation of statistics from all destroyed arenas. These constants can be
utilized either via mallctlnametomib() followed by
mallctlbymib(), or via code such as the following:
- Take special note of the
- epoch mallctl, which
- controls refreshing of cached dynamic statistics.
+ Take special note of the epoch mallctl, which controls
+ refreshing of cached dynamic statistics.
@@ -1478,6 +1482,25 @@ malloc_conf = "xmalloc:true";]]>
beforehand.
+
+
+ arena.<i>.destroy
+ (void)
+ --
+
+ Destroy the arena. Discard all of the arena's extant
+ allocations using the same mechanism as for arena.<i>.reset
+ (with all the same constraints and side effects), merge the arena stats
+ into those accessible at arena index
+ MALLCTL_ARENAS_DESTROYED, and then completely
+ discard all metadata associated with the arena. Future calls to arenas.create may
+ recycle the arena index. Destruction will fail if any threads are
+ currently associated with the arena as a result of calls to thread.arena.
+
+
arena.<i>.dss
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index 929adbe9..5e295509 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -290,6 +290,7 @@ bool arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time);
void arena_purge(tsdn_t *tsdn, arena_t *arena, bool all);
void arena_maybe_purge(tsdn_t *tsdn, arena_t *arena);
void arena_reset(tsd_t *tsd, arena_t *arena);
+void arena_destroy(tsd_t *tsd, arena_t *arena);
void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena,
tcache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes);
void arena_alloc_junk_small(void *ptr, const arena_bin_info_t *bin_info,
diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h
index 0aa82541..7dc3e5b5 100644
--- a/include/jemalloc/internal/ctl.h
+++ b/include/jemalloc/internal/ctl.h
@@ -32,7 +32,10 @@ struct ctl_indexed_node_s {
};
struct ctl_arena_stats_s {
+ unsigned arena_ind;
bool initialized;
+ ql_elm(ctl_arena_stats_t) destroyed_link;
+
unsigned nthreads;
const char *dss;
ssize_t decay_time;
@@ -62,7 +65,14 @@ struct ctl_stats_s {
size_t mapped;
size_t retained;
unsigned narenas;
- ctl_arena_stats_t *arenas[1 << MALLOCX_ARENA_BITS];
+ ql_head(ctl_arena_stats_t) destroyed;
+ /*
+ * Element 0 contains merged stats for extant arenas (accessed via
+ * MALLCTL_ARENAS_ALL), element 1 contains merged stats for destroyed
+ * arenas (accessed via MALLCTL_ARENAS_DESTROYED), and the remaining
+ * MALLOCX_ARENA_MAX+1 elements correspond to arenas.
+ */
+ ctl_arena_stats_t *arenas[MALLOCX_ARENA_MAX + 3];
};
#endif /* JEMALLOC_H_STRUCTS */
diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h
index 33b85145..70accffb 100644
--- a/include/jemalloc/internal/extent.h
+++ b/include/jemalloc/internal/extent.h
@@ -125,6 +125,8 @@ extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
+bool extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
+ extent_hooks_t **r_extent_hooks, extent_t *extent);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 991c541f..6395d750 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -215,6 +215,7 @@ typedef unsigned szind_t;
#define MALLOCX_TCACHE_SHIFT 8
#define MALLOCX_ARENA_MASK \
(((1 << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT)
+/* NB: Arena index bias decreases the maximum number of arenas by 1. */
#define MALLOCX_ARENA_MAX ((1 << MALLOCX_ARENA_BITS) - 2)
#define MALLOCX_TCACHE_MASK \
(((1 << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT)
@@ -470,6 +471,7 @@ void a0dalloc(void *ptr);
void *bootstrap_malloc(size_t size);
void *bootstrap_calloc(size_t num, size_t size);
void bootstrap_free(void *ptr);
+void arena_set(unsigned ind, arena_t *arena);
unsigned narenas_total_get(void);
arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind);
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 36960f08..c85219a9 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -21,6 +21,7 @@ arena_decay_time_default_get
arena_decay_time_default_set
arena_decay_time_get
arena_decay_time_set
+arena_destroy
arena_dss_prec_get
arena_dss_prec_set
arena_extent_alloc_large
@@ -67,6 +68,7 @@ arena_ralloc_no_move
arena_reset
arena_salloc
arena_sdalloc
+arena_set
arena_slab_regind
arena_stats_merge
arena_tcache_fill_small
@@ -164,6 +166,7 @@ extent_dalloc_cache
extent_dalloc_gap
extent_dalloc_mmap
extent_dalloc_wrapper
+extent_dalloc_wrapper_try
extent_decommit_wrapper
extent_dss_boot
extent_dss_mergeable
diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in
index ea41e2e8..05bcdd7b 100644
--- a/include/jemalloc/jemalloc_macros.h.in
+++ b/include/jemalloc/jemalloc_macros.h.in
@@ -44,6 +44,11 @@
* 0);
*/
#define MALLCTL_ARENAS_ALL 4096
+/*
+ * Use as arena index in "stats.arenas..*" mallctl interfaces to select
+ * destroyed arenas.
+ */
+#define MALLCTL_ARENAS_DESTROYED 4097
#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW)
# define JEMALLOC_CXX_THROW throw()
diff --git a/src/arena.c b/src/arena.c
index 3c31cc87..1f0c4df5 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -903,6 +903,72 @@ arena_reset(tsd_t *tsd, arena_t *arena)
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
}
+static void
+arena_destroy_retained(tsdn_t *tsdn, arena_t *arena)
+{
+ extent_hooks_t *extent_hooks = extent_hooks_get(arena);
+ size_t i;
+
+ /*
+ * Iterate over the retained extents and blindly attempt to deallocate
+ * them. This gives the extent allocator underlying the extent hooks an
+ * opportunity to unmap all retained memory without having to keep its
+ * own metadata structures, but if deallocation fails, that is the
+ * application's decision/problem. In practice, retained extents are
+ * leaked here if !config_munmap unless the application provided custom
+ * extent hooks, so best practice to either enable munmap (and avoid dss
+ * for arenas to be destroyed), or provide custom extent hooks that
+ * either unmap retained extents or track them for later use.
+ */
+ for (i = 0; i < sizeof(arena->extents_retained)/sizeof(extent_heap_t);
+ i++) {
+ extent_heap_t *extents = &arena->extents_retained[i];
+ extent_t *extent;
+
+ while ((extent = extent_heap_remove_first(extents)) != NULL) {
+ extent_dalloc_wrapper_try(tsdn, arena, &extent_hooks,
+ extent);
+ }
+ }
+}
+
+void
+arena_destroy(tsd_t *tsd, arena_t *arena)
+{
+
+ assert(base_ind_get(arena->base) >= narenas_auto);
+ assert(arena_nthreads_get(arena, false) == 0);
+ assert(arena_nthreads_get(arena, true) == 0);
+
+ /*
+ * No allocations have occurred since arena_reset() was called.
+ * Furthermore, the caller (arena_i_destroy_ctl()) purged all cached
+ * extents, so only retained extents may remain.
+ */
+ assert(arena->ndirty == 0);
+
+ /* Attempt to deallocate retained memory. */
+ arena_destroy_retained(tsd_tsdn(tsd), arena);
+
+ /*
+ * Remove the arena pointer from the arenas array. We rely on the fact
+ * that there is no way for the application to get a dirty read from the
+ * arenas array unless there is an inherent race in the application
+ * involving access of an arena being concurrently destroyed. The
+ * application must synchronize knowledge of the arena's validity, so as
+ * long as we use an atomic write to update the arenas array, the
+ * application will get a clean read any time after it synchronizes
+ * knowledge that the arena is no longer valid.
+ */
+ arena_set(base_ind_get(arena->base), NULL);
+
+ /*
+ * Destroy the base allocator, which manages all metadata ever mapped by
+ * this arena.
+ */
+ base_delete(arena->base);
+}
+
static extent_t *
arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, const arena_bin_info_t *bin_info)
diff --git a/src/ctl.c b/src/ctl.c
index 45e397b8..76fbce4b 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -48,18 +48,6 @@ static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \
const size_t *mib, size_t miblen, size_t i);
-static void ctl_arena_clear(ctl_arena_stats_t *astats);
-static void ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_stats_t *cstats,
- arena_t *arena);
-static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,
- ctl_arena_stats_t *astats);
-static void ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i);
-static bool ctl_grow(tsdn_t *tsdn, extent_hooks_t *extent_hooks);
-static void ctl_refresh(tsdn_t *tsdn);
-static bool ctl_init(tsdn_t *tsdn);
-static int ctl_lookup(tsdn_t *tsdn, const char *name,
- ctl_node_t const **nodesp, size_t *mibp, size_t *depthp);
-
CTL_PROTO(version)
CTL_PROTO(epoch)
CTL_PROTO(thread_tcache_enabled)
@@ -113,6 +101,7 @@ CTL_PROTO(arena_i_initialized)
CTL_PROTO(arena_i_purge)
CTL_PROTO(arena_i_decay)
CTL_PROTO(arena_i_reset)
+CTL_PROTO(arena_i_destroy)
CTL_PROTO(arena_i_dss)
CTL_PROTO(arena_i_decay_time)
CTL_PROTO(arena_i_extent_hooks)
@@ -274,6 +263,7 @@ static const ctl_named_node_t arena_i_node[] = {
{NAME("purge"), CTL(arena_i_purge)},
{NAME("decay"), CTL(arena_i_decay)},
{NAME("reset"), CTL(arena_i_reset)},
+ {NAME("destroy"), CTL(arena_i_destroy)},
{NAME("dss"), CTL(arena_i_dss)},
{NAME("decay_time"), CTL(arena_i_decay_time)},
{NAME("extent_hooks"), CTL(arena_i_extent_hooks)}
@@ -452,6 +442,9 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
case MALLCTL_ARENAS_ALL:
a = 0;
break;
+ case MALLCTL_ARENAS_DESTROYED:
+ a = 1;
+ break;
default:
if (compat && i == ctl_stats->narenas) {
/*
@@ -471,7 +464,7 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
*/
assert(i < ctl_stats->narenas || (!validate && i ==
ctl_stats->narenas));
- a = (unsigned)i + 1;
+ a = (unsigned)i + 2;
}
break;
}
@@ -479,6 +472,13 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
return (a);
}
+static unsigned
+stats_arenas_i2a(size_t i)
+{
+
+ return (stats_arenas_i2a_impl(i, true, false));
+}
+
static ctl_arena_stats_t *
stats_arenas_i_impl(tsdn_t *tsdn, size_t i, bool compat, bool init)
{
@@ -492,10 +492,13 @@ stats_arenas_i_impl(tsdn_t *tsdn, size_t i, bool compat, bool init)
sizeof(ctl_arena_stats_t), QUANTUM);
if (ret == NULL)
return (NULL);
+ ret->arena_ind = (unsigned)i;
ctl_stats->arenas[stats_arenas_i2a_impl(i, compat, false)] =
ret;
}
+ assert(ret == NULL || stats_arenas_i2a(ret->arena_ind) ==
+ stats_arenas_i2a(i));
return (ret);
}
@@ -553,92 +556,130 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_stats_t *cstats, arena_t *arena)
}
static void
-ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
+ctl_arena_stats_sdmerge(ctl_arena_stats_t *sdstats, ctl_arena_stats_t *astats,
+ bool destroyed)
{
unsigned i;
- sstats->nthreads += astats->nthreads;
- sstats->pactive += astats->pactive;
- sstats->pdirty += astats->pdirty;
+ if (!destroyed) {
+ sdstats->nthreads += astats->nthreads;
+ sdstats->pactive += astats->pactive;
+ sdstats->pdirty += astats->pdirty;
+ } else {
+ assert(astats->nthreads == 0);
+ assert(astats->pactive == 0);
+ assert(astats->pdirty == 0);
+ }
if (config_stats) {
- sstats->astats.mapped += astats->astats.mapped;
- sstats->astats.retained += astats->astats.retained;
- sstats->astats.npurge += astats->astats.npurge;
- sstats->astats.nmadvise += astats->astats.nmadvise;
- sstats->astats.purged += astats->astats.purged;
+ if (!destroyed) {
+ sdstats->astats.mapped += astats->astats.mapped;
+ sdstats->astats.retained += astats->astats.retained;
+ }
+ sdstats->astats.npurge += astats->astats.npurge;
+ sdstats->astats.nmadvise += astats->astats.nmadvise;
+ sdstats->astats.purged += astats->astats.purged;
- sstats->astats.base += astats->astats.base;
- sstats->astats.internal += astats->astats.internal;
- sstats->astats.resident += astats->astats.resident;
+ if (!destroyed) {
+ sdstats->astats.base += astats->astats.base;
+ sdstats->astats.internal += astats->astats.internal;
+ sdstats->astats.resident += astats->astats.resident;
+ } else
+ assert(astats->astats.internal == 0);
- sstats->allocated_small += astats->allocated_small;
- sstats->nmalloc_small += astats->nmalloc_small;
- sstats->ndalloc_small += astats->ndalloc_small;
- sstats->nrequests_small += astats->nrequests_small;
+ if (!destroyed)
+ sdstats->allocated_small += astats->allocated_small;
+ else
+ assert(astats->allocated_small == 0);
+ sdstats->nmalloc_small += astats->nmalloc_small;
+ sdstats->ndalloc_small += astats->ndalloc_small;
+ sdstats->nrequests_small += astats->nrequests_small;
- sstats->astats.allocated_large +=
- astats->astats.allocated_large;
- sstats->astats.nmalloc_large += astats->astats.nmalloc_large;
- sstats->astats.ndalloc_large += astats->astats.ndalloc_large;
- sstats->astats.nrequests_large +=
+ if (!destroyed) {
+ sdstats->astats.allocated_large +=
+ astats->astats.allocated_large;
+ } else
+ assert(astats->astats.allocated_large == 0);
+ sdstats->astats.nmalloc_large += astats->astats.nmalloc_large;
+ sdstats->astats.ndalloc_large += astats->astats.ndalloc_large;
+ sdstats->astats.nrequests_large +=
astats->astats.nrequests_large;
for (i = 0; i < NBINS; i++) {
- sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
- sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc;
- sstats->bstats[i].nrequests +=
+ sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
+ sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc;
+ sdstats->bstats[i].nrequests +=
astats->bstats[i].nrequests;
- sstats->bstats[i].curregs += astats->bstats[i].curregs;
+ if (!destroyed) {
+ sdstats->bstats[i].curregs +=
+ astats->bstats[i].curregs;
+ } else
+ assert(astats->bstats[i].curregs == 0);
if (config_tcache) {
- sstats->bstats[i].nfills +=
+ sdstats->bstats[i].nfills +=
astats->bstats[i].nfills;
- sstats->bstats[i].nflushes +=
+ sdstats->bstats[i].nflushes +=
astats->bstats[i].nflushes;
}
- sstats->bstats[i].nslabs += astats->bstats[i].nslabs;
- sstats->bstats[i].reslabs += astats->bstats[i].reslabs;
- sstats->bstats[i].curslabs +=
- astats->bstats[i].curslabs;
+ sdstats->bstats[i].nslabs += astats->bstats[i].nslabs;
+ sdstats->bstats[i].reslabs += astats->bstats[i].reslabs;
+ if (!destroyed) {
+ sdstats->bstats[i].curslabs +=
+ astats->bstats[i].curslabs;
+ } else
+ assert(astats->bstats[i].curslabs == 0);
}
for (i = 0; i < NSIZES - NBINS; i++) {
- sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc;
- sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc;
- sstats->lstats[i].nrequests +=
+ sdstats->lstats[i].nmalloc += astats->lstats[i].nmalloc;
+ sdstats->lstats[i].ndalloc += astats->lstats[i].ndalloc;
+ sdstats->lstats[i].nrequests +=
astats->lstats[i].nrequests;
- sstats->lstats[i].curlextents +=
- astats->lstats[i].curlextents;
+ if (!destroyed) {
+ sdstats->lstats[i].curlextents +=
+ astats->lstats[i].curlextents;
+ } else
+ assert(astats->lstats[i].curlextents == 0);
}
}
}
static void
-ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i)
+ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_stats_t *sdstats,
+ unsigned i, bool destroyed)
{
ctl_arena_stats_t *astats = stats_arenas_i(i);
- ctl_arena_stats_t *sstats = stats_arenas_i(MALLCTL_ARENAS_ALL);
ctl_arena_clear(astats);
ctl_arena_stats_amerge(tsdn, astats, arena);
/* Merge into sum stats as well. */
- ctl_arena_stats_smerge(sstats, astats);
+ ctl_arena_stats_sdmerge(sdstats, astats, destroyed);
}
-static bool
-ctl_grow(tsdn_t *tsdn, extent_hooks_t *extent_hooks)
+static unsigned
+ctl_arena_init(tsdn_t *tsdn, extent_hooks_t *extent_hooks)
{
+ unsigned arena_ind;
+ ctl_arena_stats_t *astats;
+
+ if ((astats = ql_last(&ctl_stats->destroyed, destroyed_link)) != NULL) {
+ ql_remove(&ctl_stats->destroyed, astats, destroyed_link);
+ arena_ind = astats->arena_ind;
+ } else
+ arena_ind = ctl_stats->narenas;
/* Trigger stats allocation. */
- if (stats_arenas_i_impl(tsdn, ctl_stats->narenas, false, true) == NULL)
- return (true);
+ if (stats_arenas_i_impl(tsdn, arena_ind, false, true) == NULL)
+ return (UINT_MAX);
/* Initialize new arena. */
- if (arena_init(tsdn, ctl_stats->narenas, extent_hooks) == NULL)
- return (true);
- ctl_stats->narenas++;
+ if (arena_init(tsdn, arena_ind, extent_hooks) == NULL)
+ return (UINT_MAX);
- return (false);
+ if (arena_ind == ctl_stats->narenas)
+ ctl_stats->narenas++;
+
+ return (arena_ind);
}
static void
@@ -663,7 +704,7 @@ ctl_refresh(tsdn_t *tsdn)
astats->initialized = initialized;
if (initialized)
- ctl_arena_refresh(tsdn, tarenas[i], i);
+ ctl_arena_refresh(tsdn, tarenas[i], sstats, i, false);
}
if (config_stats) {
@@ -687,7 +728,7 @@ ctl_init(tsdn_t *tsdn)
malloc_mutex_lock(tsdn, &ctl_mtx);
if (!ctl_initialized) {
- ctl_arena_stats_t *sstats;
+ ctl_arena_stats_t *sstats, *dstats;
unsigned i;
/*
@@ -715,6 +756,19 @@ ctl_init(tsdn_t *tsdn)
}
sstats->initialized = true;
+ if ((dstats = stats_arenas_i_impl(tsdn,
+ MALLCTL_ARENAS_DESTROYED, false, true)) == NULL) {
+ ret = true;
+ goto label_return;
+ }
+ ctl_arena_clear(dstats);
+ /*
+ * Don't toggle stats for MALLCTL_ARENAS_DESTROYED to
+ * initialized until an arena is actually destroyed, so that
+ * arena..initialized can be used to query whether the stats
+ * are relevant.
+ */
+
ctl_stats->narenas = narenas_total_get();
for (i = 0; i < ctl_stats->narenas; i++) {
if (stats_arenas_i_impl(tsdn, i, false, true) == NULL) {
@@ -723,7 +777,7 @@ ctl_init(tsdn_t *tsdn)
}
}
- ctl_stats->epoch = 0;
+ ql_new(&ctl_stats->destroyed);
ctl_refresh(tsdn);
ctl_initialized = true;
}
@@ -1562,6 +1616,33 @@ label_return:
return (ret);
}
+static int
+arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen,
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind,
+ arena_t **arena)
+{
+ int ret;
+
+ READONLY();
+ WRITEONLY();
+ MIB_UNSIGNED(*arena_ind, 1);
+
+ if (*arena_ind < narenas_auto) {
+ ret = EFAULT;
+ goto label_return;
+ }
+
+ *arena = arena_get(tsd_tsdn(tsd), *arena_ind, false);
+ if (*arena == NULL) {
+ ret = EFAULT;
+ goto label_return;
+ }
+
+ ret = 0;
+label_return:
+ return (ret);
+}
+
static int
arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
@@ -1570,26 +1651,51 @@ arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
unsigned arena_ind;
arena_t *arena;
- READONLY();
- WRITEONLY();
- MIB_UNSIGNED(arena_ind, 1);
+ ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
+ newp, newlen, &arena_ind, &arena);
+ if (ret != 0)
+ return (ret);
- if (config_debug) {
- malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
- assert(arena_ind < ctl_stats->narenas);
- malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
- }
- assert(arena_ind >= opt_narenas);
+ arena_reset(tsd, arena);
- arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
- if (arena == NULL) {
+ return (ret);
+}
+
+static int
+arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
+{
+ int ret;
+ unsigned arena_ind;
+ arena_t *arena;
+ ctl_arena_stats_t *dstats, *astats;
+
+ ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
+ newp, newlen, &arena_ind, &arena);
+ if (ret != 0)
+ goto label_return;
+
+ if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena,
+ true) != 0) {
ret = EFAULT;
goto label_return;
}
+ /* Merge stats after resetting and purging arena. */
arena_reset(tsd, arena);
+ arena_purge(tsd_tsdn(tsd), arena, true);
+ dstats = stats_arenas_i(MALLCTL_ARENAS_DESTROYED);
+ dstats->initialized = true;
+ ctl_arena_refresh(tsd_tsdn(tsd), arena, dstats, arena_ind, true);
+ /* Destroy arena. */
+ arena_destroy(tsd, arena);
+ astats = stats_arenas_i(arena_ind);
+ astats->initialized = false;
+ /* Record arena index for later recycling via arenas.create. */
+ ql_elm_new(astats, destroyed_link);
+ ql_tail_insert(&ctl_stats->destroyed, astats, destroyed_link);
- ret = 0;
+ assert(ret == 0);
label_return:
return (ret);
}
@@ -1733,9 +1839,16 @@ arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
const ctl_named_node_t *ret;
malloc_mutex_lock(tsdn, &ctl_mtx);
- if (i > ctl_stats->narenas && i != MALLCTL_ARENAS_ALL) {
- ret = NULL;
- goto label_return;
+ switch (i) {
+ case MALLCTL_ARENAS_ALL:
+ case MALLCTL_ARENAS_DESTROYED:
+ break;
+ default:
+ if (i > ctl_stats->narenas) {
+ ret = NULL;
+ goto label_return;
+ }
+ break;
}
ret = super_arena_i_node;
@@ -1828,18 +1941,18 @@ arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
{
int ret;
extent_hooks_t *extent_hooks;
- unsigned narenas;
+ unsigned arena_ind;
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
extent_hooks = (extent_hooks_t *)&extent_hooks_default;
WRITE(extent_hooks, extent_hooks_t *);
- if (ctl_grow(tsd_tsdn(tsd), extent_hooks)) {
+ if ((arena_ind = ctl_arena_init(tsd_tsdn(tsd), extent_hooks)) ==
+ UINT_MAX) {
ret = EAGAIN;
goto label_return;
}
- narenas = ctl_stats->narenas - 1;
- READ(narenas, unsigned);
+ READ(arena_ind, unsigned);
ret = 0;
label_return:
diff --git a/src/extent.c b/src/extent.c
index 6eabde31..7eb49709 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -1039,11 +1039,11 @@ extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
return (extent_dalloc_default_impl(addr, size));
}
-void
-extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
+bool
+extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent)
{
- bool err, zeroed;
+ bool err;
assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0);
@@ -1067,10 +1067,21 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_committed_get(extent), arena_ind_get(arena)));
}
- if (!err) {
+ if (!err)
extent_dalloc(tsdn, arena, extent);
+
+ return (err);
+}
+
+void
+extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
+ extent_hooks_t **r_extent_hooks, extent_t *extent)
+{
+ bool zeroed;
+
+ if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent))
return;
- }
+
extent_reregister(tsdn, extent);
/* Try to decommit; purge if that fails. */
if (!extent_committed_get(extent))
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 2acab412..a053983f 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -375,7 +375,7 @@ bootstrap_free(void *ptr)
a0idalloc(iealloc(NULL, ptr), ptr, false);
}
-static void
+void
arena_set(unsigned ind, arena_t *arena)
{
diff --git a/src/stats.c b/src/stats.c
index 4e09eb45..ef349a50 100644
--- a/src/stats.c
+++ b/src/stats.c
@@ -772,7 +772,8 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
static void
stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
- bool json, bool merged, bool unmerged, bool bins, bool large)
+ bool json, bool merged, bool destroyed, bool unmerged, bool bins,
+ bool large)
{
size_t allocated, active, metadata, resident, mapped, retained;
@@ -808,7 +809,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
allocated, active, metadata, resident, mapped, retained);
}
- if (merged || unmerged) {
+ if (merged || destroyed || unmerged) {
unsigned narenas;
if (json) {
@@ -822,6 +823,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
size_t miblen = sizeof(mib) / sizeof(size_t);
size_t sz;
VARIABLE_ARRAY(bool, initialized, narenas);
+ bool destroyed_initialized;
unsigned i, j, ninitialized;
xmallctlnametomib("arena.0.initialized", mib, &miblen);
@@ -833,6 +835,10 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
if (initialized[i])
ninitialized++;
}
+ mib[1] = MALLCTL_ARENAS_DESTROYED;
+ sz = sizeof(bool);
+ xmallctlbymib(mib, miblen, &destroyed_initialized, &sz,
+ NULL, 0);
/* Merged stats. */
if (merged && (ninitialized > 1 || !unmerged)) {
@@ -853,6 +859,25 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
}
}
+ /* Destroyed stats. */
+ if (destroyed_initialized && destroyed) {
+ /* Print destroyed arena stats. */
+ if (json) {
+ malloc_cprintf(write_cb, cbopaque,
+ "\t\t\t\"destroyed\": {\n");
+ } else {
+ malloc_cprintf(write_cb, cbopaque,
+ "\nDestroyed arenas stats:\n");
+ }
+ stats_arena_print(write_cb, cbopaque, json,
+ MALLCTL_ARENAS_DESTROYED, bins, large);
+ if (json) {
+ malloc_cprintf(write_cb, cbopaque,
+ "\t\t\t}%s\n", (ninitialized > 1) ?
+ "," : "");
+ }
+ }
+
/* Unmerged stats. */
for (i = j = 0; i < narenas; i++) {
if (initialized[i]) {
@@ -895,6 +920,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool json = false;
bool general = true;
bool merged = config_stats;
+ bool destroyed = config_stats;
bool unmerged = config_stats;
bool bins = true;
bool large = true;
@@ -935,6 +961,9 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
case 'm':
merged = false;
break;
+ case 'd':
+ destroyed = false;
+ break;
case 'a':
unmerged = false;
break;
@@ -963,8 +992,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
stats_general_print(write_cb, cbopaque, json, more);
}
if (config_stats) {
- stats_print_helper(write_cb, cbopaque, json, merged, unmerged,
- bins, large);
+ stats_print_helper(write_cb, cbopaque, json, merged, destroyed,
+ unmerged, bins, large);
}
if (json) {
diff --git a/test/unit/arena_reset.c b/test/unit/arena_reset.c
index 3a1b30f5..65ff1031 100644
--- a/test/unit/arena_reset.c
+++ b/test/unit/arena_reset.c
@@ -1,9 +1,9 @@
+#ifndef ARENA_RESET_PROF_C_
#include "test/jemalloc_test.h"
-
-#ifdef JEMALLOC_PROF
-const char *malloc_conf = "prof:true,lg_prof_sample:0";
#endif
+#include "test/extent_hooks.h"
+
static unsigned
get_nsizes_impl(const char *cmd)
{
@@ -79,57 +79,64 @@ vsalloc(tsdn_t *tsdn, const void *ptr)
return (isalloc(tsdn, extent, ptr));
}
-TEST_BEGIN(test_arena_reset)
+static unsigned
+do_arena_create(extent_hooks_t *h)
+{
+ unsigned arena_ind;
+ size_t sz = sizeof(unsigned);
+ assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz,
+ (void *)(h != NULL ? &h : NULL), (h != NULL ? sizeof(h) : 0)), 0,
+ "Unexpected mallctl() failure");
+ return (arena_ind);
+}
+
+static void
+do_arena_reset_pre(unsigned arena_ind, void ***ptrs, unsigned *nptrs)
{
#define NLARGE 32
- unsigned arena_ind, nsmall, nlarge, nptrs, i;
- size_t sz, miblen;
- void **ptrs;
+ unsigned nsmall, nlarge, i;
+ size_t sz;
int flags;
- size_t mib[3];
tsdn_t *tsdn;
- sz = sizeof(unsigned);
- assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
- 0, "Unexpected mallctl() failure");
-
flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
nsmall = get_nsmall();
nlarge = get_nlarge() > NLARGE ? NLARGE : get_nlarge();
- nptrs = nsmall + nlarge;
- ptrs = (void **)malloc(nptrs * sizeof(void *));
- assert_ptr_not_null(ptrs, "Unexpected malloc() failure");
+ *nptrs = nsmall + nlarge;
+ *ptrs = (void **)malloc(*nptrs * sizeof(void *));
+ assert_ptr_not_null(*ptrs, "Unexpected malloc() failure");
/* Allocate objects with a wide range of sizes. */
for (i = 0; i < nsmall; i++) {
sz = get_small_size(i);
- ptrs[i] = mallocx(sz, flags);
- assert_ptr_not_null(ptrs[i],
+ (*ptrs)[i] = mallocx(sz, flags);
+ assert_ptr_not_null((*ptrs)[i],
"Unexpected mallocx(%zu, %#x) failure", sz, flags);
}
for (i = 0; i < nlarge; i++) {
sz = get_large_size(i);
- ptrs[nsmall + i] = mallocx(sz, flags);
- assert_ptr_not_null(ptrs[i],
+ (*ptrs)[nsmall + i] = mallocx(sz, flags);
+ assert_ptr_not_null((*ptrs)[i],
"Unexpected mallocx(%zu, %#x) failure", sz, flags);
}
tsdn = tsdn_fetch();
/* Verify allocations. */
- for (i = 0; i < nptrs; i++) {
- assert_zu_gt(ivsalloc(tsdn, ptrs[i]), 0,
+ for (i = 0; i < *nptrs; i++) {
+ assert_zu_gt(ivsalloc(tsdn, (*ptrs)[i]), 0,
"Allocation should have queryable size");
}
+}
- /* Reset. */
- miblen = sizeof(mib)/sizeof(size_t);
- assert_d_eq(mallctlnametomib("arena.0.reset", mib, &miblen), 0,
- "Unexpected mallctlnametomib() failure");
- mib[1] = (size_t)arena_ind;
- assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
- "Unexpected mallctlbymib() failure");
+static void
+do_arena_reset_post(void **ptrs, unsigned nptrs)
+{
+ tsdn_t *tsdn;
+ unsigned i;
+
+ tsdn = tsdn_fetch();
/* Verify allocations no longer exist. */
for (i = 0; i < nptrs; i++) {
@@ -139,6 +146,193 @@ TEST_BEGIN(test_arena_reset)
free(ptrs);
}
+
+static void
+do_arena_reset_destroy(const char *name, unsigned arena_ind)
+{
+ size_t mib[3];
+ size_t miblen;
+
+ miblen = sizeof(mib)/sizeof(size_t);
+ assert_d_eq(mallctlnametomib(name, mib, &miblen), 0,
+ "Unexpected mallctlnametomib() failure");
+ mib[1] = (size_t)arena_ind;
+ assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
+ "Unexpected mallctlbymib() failure");
+}
+
+static void
+do_arena_reset(unsigned arena_ind)
+{
+
+ do_arena_reset_destroy("arena.0.reset", arena_ind);
+}
+
+static void
+do_arena_destroy(unsigned arena_ind)
+{
+
+ do_arena_reset_destroy("arena.0.destroy", arena_ind);
+}
+
+TEST_BEGIN(test_arena_reset)
+{
+ unsigned arena_ind;
+ void **ptrs;
+ unsigned nptrs;
+
+ arena_ind = do_arena_create(NULL);
+ do_arena_reset_pre(arena_ind, &ptrs, &nptrs);
+ do_arena_reset(arena_ind);
+ do_arena_reset_post(ptrs, nptrs);
+}
+TEST_END
+
+static bool
+arena_i_initialized(unsigned arena_ind, bool refresh)
+{
+ bool initialized;
+ size_t mib[3];
+ size_t miblen, sz;
+
+ if (refresh) {
+ uint64_t epoch = 1;
+ assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch,
+ sizeof(epoch)), 0, "Unexpected mallctl() failure");
+ }
+
+ miblen = sizeof(mib)/sizeof(size_t);
+ assert_d_eq(mallctlnametomib("arena.0.initialized", mib, &miblen), 0,
+ "Unexpected mallctlnametomib() failure");
+ mib[1] = (size_t)arena_ind;
+ sz = sizeof(initialized);
+ assert_d_eq(mallctlbymib(mib, miblen, (void *)&initialized, &sz, NULL,
+ 0), 0, "Unexpected mallctlbymib() failure");
+
+ return (initialized);
+}
+
+TEST_BEGIN(test_arena_destroy_initial)
+{
+
+ assert_false(arena_i_initialized(MALLCTL_ARENAS_DESTROYED, false),
+ "Destroyed arena stats should not be initialized");
+}
+TEST_END
+
+TEST_BEGIN(test_arena_destroy_hooks_default)
+{
+ unsigned arena_ind, arena_ind_another, arena_ind_prev;
+ void **ptrs;
+ unsigned nptrs;
+
+ arena_ind = do_arena_create(NULL);
+ do_arena_reset_pre(arena_ind, &ptrs, &nptrs);
+
+ assert_false(arena_i_initialized(arena_ind, false),
+ "Arena stats should not be initialized");
+ assert_true(arena_i_initialized(arena_ind, true),
+ "Arena stats should be initialized");
+
+ /*
+ * Create another arena before destroying one, to better verify arena
+ * index reuse.
+ */
+ arena_ind_another = do_arena_create(NULL);
+
+ do_arena_destroy(arena_ind);
+
+ assert_false(arena_i_initialized(arena_ind, true),
+ "Arena stats should not be initialized");
+ assert_true(arena_i_initialized(MALLCTL_ARENAS_DESTROYED, false),
+ "Destroyed arena stats should be initialized");
+
+ do_arena_reset_post(ptrs, nptrs);
+
+ arena_ind_prev = arena_ind;
+ arena_ind = do_arena_create(NULL);
+ do_arena_reset_pre(arena_ind, &ptrs, &nptrs);
+ assert_u_eq(arena_ind, arena_ind_prev,
+ "Arena index should have been recycled");
+ do_arena_destroy(arena_ind);
+ do_arena_reset_post(ptrs, nptrs);
+
+ do_arena_destroy(arena_ind_another);
+}
+TEST_END
+
+/*
+ * Actually unmap extents, regardless of config_munmap, so that attempts to
+ * access a destroyed arena's memory will segfault.
+ */
+static bool
+extent_dalloc_unmap(extent_hooks_t *extent_hooks, void *addr, size_t size,
+ bool committed, unsigned arena_ind)
+{
+
+ TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, committed=%s, "
+ "arena_ind=%u)\n", __func__, extent_hooks, addr, size, committed ?
+ "true" : "false", arena_ind);
+ assert_ptr_eq(extent_hooks, &hooks,
+ "extent_hooks should be same as pointer used to set hooks");
+ assert_ptr_eq(extent_hooks->dalloc, extent_dalloc_unmap,
+ "Wrong hook function");
+ called_dalloc = true;
+ if (!try_dalloc)
+ return (true);
+ pages_unmap(addr, size);
+ did_dalloc = true;
+ return (false);
+}
+
+static extent_hooks_t hooks_orig;
+
+static extent_hooks_t hooks_unmap = {
+ extent_alloc_hook,
+ extent_dalloc_unmap, /* dalloc */
+ extent_commit_hook,
+ extent_decommit_hook,
+ extent_purge_lazy_hook,
+ extent_purge_forced_hook,
+ extent_split_hook,
+ extent_merge_hook
+};
+
+TEST_BEGIN(test_arena_destroy_hooks_unmap)
+{
+ unsigned arena_ind;
+ void **ptrs;
+ unsigned nptrs;
+
+ extent_hooks_prep();
+ try_decommit = false;
+ memcpy(&hooks_orig, &hooks, sizeof(extent_hooks_t));
+ memcpy(&hooks, &hooks_unmap, sizeof(extent_hooks_t));
+
+ did_alloc = false;
+ arena_ind = do_arena_create(&hooks);
+ do_arena_reset_pre(arena_ind, &ptrs, &nptrs);
+
+ assert_true(did_alloc, "Expected alloc");
+
+ assert_false(arena_i_initialized(arena_ind, false),
+ "Arena stats should not be initialized");
+ assert_true(arena_i_initialized(arena_ind, true),
+ "Arena stats should be initialized");
+
+ did_dalloc = false;
+ do_arena_destroy(arena_ind);
+ assert_true(did_dalloc, "Expected dalloc");
+
+ assert_false(arena_i_initialized(arena_ind, true),
+ "Arena stats should not be initialized");
+ assert_true(arena_i_initialized(MALLCTL_ARENAS_DESTROYED, false),
+ "Destroyed arena stats should be initialized");
+
+ do_arena_reset_post(ptrs, nptrs);
+
+ memcpy(&hooks, &hooks_orig, sizeof(extent_hooks_t));
+}
TEST_END
int
@@ -146,5 +340,8 @@ main(void)
{
return (test(
- test_arena_reset));
+ test_arena_reset,
+ test_arena_destroy_initial,
+ test_arena_destroy_hooks_default,
+ test_arena_destroy_hooks_unmap));
}
diff --git a/test/unit/arena_reset_prof.c b/test/unit/arena_reset_prof.c
new file mode 100644
index 00000000..0fd362e9
--- /dev/null
+++ b/test/unit/arena_reset_prof.c
@@ -0,0 +1,5 @@
+#include "test/jemalloc_test.h"
+#define ARENA_RESET_PROF_C_
+
+const char *malloc_conf = "prof:true,lg_prof_sample:0";
+#include "arena_reset.c"
diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c
index b3320788..fbe76cb4 100644
--- a/test/unit/mallctl.c
+++ b/test/unit/mallctl.c
@@ -381,6 +381,15 @@ TEST_BEGIN(test_arena_i_initialized)
"Unexpected mallctl() failure");
assert_true(initialized,
"Merged arena statistics should always be initialized");
+
+ /* Equivalent to the above but using mallctl() directly. */
+ sz = sizeof(initialized);
+ assert_d_eq(mallctl(
+ "arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".initialized",
+ (void *)&initialized, &sz, NULL, 0), 0,
+ "Unexpected mallctl() failure");
+ assert_true(initialized,
+ "Merged arena statistics should always be initialized");
}
TEST_END