diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index d6b1a2b0..afa8984d 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -232,6 +232,15 @@ struct arena_s {
void *extent_hooks_pun;
};
+ /*
+ * Next extent size class in a growing series to use when satisfying a
+ * request via the extent hooks (only if !config_munmap). This limits
+ * the number of disjoint virtual memory ranges so that extent merging
+ * can be effective even if multiple arenas' extent allocation requests
+ * are highly interleaved.
+ */
+ pszind_t extent_grow_next;
+
/* Cache of extent structures that were allocated via base_alloc(). */
ql_head(extent_t) extent_cache;
malloc_mutex_t extent_cache_mtx;
diff --git a/src/arena.c b/src/arena.c
index 73fea528..c3587044 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -1686,6 +1686,9 @@ arena_new(tsdn_t *tsdn, unsigned ind)
arena->extent_hooks = (extent_hooks_t *)&extent_hooks_default;
+ if (!config_munmap)
+ arena->extent_grow_next = psz2ind(HUGEPAGE);
+
ql_new(&arena->extent_cache);
if (malloc_mutex_init(&arena->extent_cache_mtx, "arena_extent_cache",
WITNESS_RANK_ARENA_EXTENT_CACHE))
diff --git a/src/extent.c b/src/extent.c
index be6cadc3..586e8d33 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -265,6 +265,41 @@ extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
}
}
+static void
+extent_gprof_add(tsdn_t *tsdn, const extent_t *extent)
+{
+
+ cassert(config_prof);
+
+ if (opt_prof && extent_active_get(extent)) {
+ size_t nadd = extent_size_get(extent) >> LG_PAGE;
+ size_t cur = atomic_add_zu(&curpages, nadd);
+ size_t high = atomic_read_zu(&highpages);
+ while (cur > high && atomic_cas_zu(&highpages, high, cur)) {
+ /*
+ * Don't refresh cur, because it may have decreased
+ * since this thread lost the highpages update race.
+ */
+ high = atomic_read_zu(&highpages);
+ }
+ if (cur > high && prof_gdump_get_unlocked())
+ prof_gdump(tsdn);
+ }
+}
+
+static void
+extent_gprof_sub(tsdn_t *tsdn, const extent_t *extent)
+{
+
+ cassert(config_prof);
+
+ if (opt_prof && extent_active_get(extent)) {
+ size_t nsub = extent_size_get(extent) >> LG_PAGE;
+ assert(atomic_read_zu(&curpages) >= nsub);
+ atomic_sub_zu(&curpages, nsub);
+ }
+}
+
static bool
extent_register(tsdn_t *tsdn, const extent_t *extent)
{
@@ -280,20 +315,8 @@ extent_register(tsdn_t *tsdn, const extent_t *extent)
extent_interior_register(tsdn, rtree_ctx, extent);
extent_rtree_release(tsdn, elm_a, elm_b);
- if (config_prof && opt_prof && extent_active_get(extent)) {
- size_t nadd = extent_size_get(extent) >> LG_PAGE;
- size_t cur = atomic_add_zu(&curpages, nadd);
- size_t high = atomic_read_zu(&highpages);
- while (cur > high && atomic_cas_zu(&highpages, high, cur)) {
- /*
- * Don't refresh cur, because it may have decreased
- * since this thread lost the highpages update race.
- */
- high = atomic_read_zu(&highpages);
- }
- if (cur > high && prof_gdump_get_unlocked())
- prof_gdump(tsdn);
- }
+ if (config_prof)
+ extent_gprof_add(tsdn, extent);
return (false);
}
@@ -336,11 +359,8 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent)
}
extent_rtree_release(tsdn, elm_a, elm_b);
- if (config_prof && opt_prof && extent_active_get(extent)) {
- size_t nsub = extent_size_get(extent) >> LG_PAGE;
- assert(atomic_read_zu(&curpages) >= nsub);
- atomic_sub_zu(&curpages, nsub);
- }
+ if (config_prof)
+ extent_gprof_sub(tsdn, extent);
}
/*
@@ -507,14 +527,16 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_usize_set(extent, usize);
}
- if (commit && !extent_committed_get(extent) &&
- extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent, 0,
- extent_size_get(extent))) {
- if (!locked)
- malloc_mutex_unlock(tsdn, &arena->extents_mtx);
- extent_record(tsdn, arena, r_extent_hooks, extent_heaps, cache,
- extent);
- return (NULL);
+ if (*commit && !extent_committed_get(extent)) {
+ if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent,
+ 0, extent_size_get(extent))) {
+ if (!locked)
+ malloc_mutex_unlock(tsdn, &arena->extents_mtx);
+ extent_record(tsdn, arena, r_extent_hooks, extent_heaps,
+ cache, extent);
+ return (NULL);
+ }
+ extent_zeroed_set(extent, true);
}
if (pad != 0)
@@ -591,8 +613,6 @@ extent_alloc_cache_impl(tsdn_t *tsdn, arena_t *arena,
extent = extent_recycle(tsdn, arena, r_extent_hooks,
arena->extents_cached, locked, true, new_addr, usize, pad,
alignment, zero, commit, slab);
- if (extent == NULL)
- return (NULL);
return (extent);
}
@@ -626,9 +646,6 @@ extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,
ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
commit, arena->dss_prec);
- if (ret == NULL)
- return (NULL);
-
return (ret);
}
@@ -653,6 +670,136 @@ extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
alignment, zero, commit));
}
+static void
+extent_retain(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
+ extent_t *extent)
+{
+
+ if (config_stats)
+ arena->stats.retained += extent_size_get(extent);
+ extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained,
+ false, extent);
+}
+
+/*
+ * If virtual memory is retained, create increasingly larger extents from which
+ * to split requested extents in order to limit the total number of disjoint
+ * virtual memory ranges retained by each arena.
+ */
+static extent_t *
+extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
+ extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
+ size_t alignment, bool *zero, bool *commit, bool slab)
+{
+ extent_t *extent;
+ void *ptr;
+ size_t size, alloc_size, alloc_size_min, leadsize, trailsize;
+ bool zeroed, committed;
+
+ /*
+ * Check whether the next extent size in the series would be large
+ * enough to satisfy this request. If no, just bail, so that e.g. a
+ * series of unsatisfiable allocation requests doesn't cause unused
+ * extent creation as a side effect.
+ */
+ size = usize + pad;
+ alloc_size = pind2sz(arena->extent_grow_next);
+ alloc_size_min = size + PAGE_CEILING(alignment) - PAGE;
+ /* Beware size_t wrap-around. */
+ if (alloc_size_min < usize)
+ return (NULL);
+ if (alloc_size < alloc_size_min)
+ return (NULL);
+ extent = extent_alloc(tsdn, arena);
+ if (extent == NULL)
+ return (NULL);
+ zeroed = false;
+ committed = false;
+ ptr = extent_alloc_core(tsdn, arena, new_addr, alloc_size, PAGE,
+ &zeroed, &committed, arena->dss_prec);
+ extent_init(extent, arena, ptr, alloc_size, alloc_size,
+ arena_extent_sn_next(arena), false, zeroed, committed, false);
+ if (ptr == NULL || extent_register(tsdn, extent)) {
+ extent_dalloc(tsdn, arena, extent);
+ return (NULL);
+ }
+ /*
+ * Set the extent as active *after registration so that no gprof-related
+ * accounting occurs during registration.
+ */
+ extent_active_set(extent, true);
+
+ leadsize = ALIGNMENT_CEILING((uintptr_t)ptr, PAGE_CEILING(alignment)) -
+ (uintptr_t)ptr;
+ assert(new_addr == NULL || leadsize == 0);
+ assert(alloc_size >= leadsize + size);
+ trailsize = alloc_size - leadsize - size;
+ if (extent_zeroed_get(extent))
+ *zero = true;
+ if (extent_committed_get(extent))
+ *commit = true;
+
+ /* Split the lead. */
+ if (leadsize != 0) {
+ extent_t *lead = extent;
+ extent = extent_split_wrapper(tsdn, arena, r_extent_hooks, lead,
+ leadsize, leadsize, size + trailsize, usize + trailsize);
+ if (extent == NULL) {
+ extent_deregister(tsdn, lead);
+ extent_leak(tsdn, arena, r_extent_hooks, false, lead);
+ return (NULL);
+ }
+ extent_retain(tsdn, arena, r_extent_hooks, lead);
+ }
+
+ /* Split the trail. */
+ if (trailsize != 0) {
+ extent_t *trail = extent_split_wrapper(tsdn, arena,
+ r_extent_hooks, extent, size, usize, trailsize, trailsize);
+ if (trail == NULL) {
+ extent_deregister(tsdn, extent);
+ extent_leak(tsdn, arena, r_extent_hooks, false, extent);
+ return (NULL);
+ }
+ extent_retain(tsdn, arena, r_extent_hooks, trail);
+ } else if (leadsize == 0) {
+ /*
+ * Splitting causes usize to be set as a side effect, but no
+ * splitting occurred.
+ */
+ extent_usize_set(extent, usize);
+ }
+
+ if (*commit && !extent_committed_get(extent)) {
+ if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent,
+ 0, extent_size_get(extent))) {
+ extent_retain(tsdn, arena, r_extent_hooks, extent);
+ return (NULL);
+ }
+ extent_zeroed_set(extent, true);
+ }
+
+ if (config_prof) {
+ /* Adjust gprof stats now that extent is final size. */
+ extent_gprof_add(tsdn, extent);
+ }
+ if (pad != 0)
+ extent_addr_randomize(tsdn, extent, alignment);
+ if (slab) {
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
+ &rtree_ctx_fallback);
+
+ extent_slab_set(extent, true);
+ extent_interior_register(tsdn, rtree_ctx, extent);
+ }
+ if (*zero && !extent_zeroed_get(extent))
+ memset(extent_addr_get(extent), 0, extent_usize_get(extent));
+ if (arena->extent_grow_next + 1 < NPSIZES)
+ arena->extent_grow_next++;
+ return (extent);
+}
+
static extent_t *
extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
@@ -669,6 +816,12 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
if (extent != NULL && config_stats) {
size_t size = usize + pad;
arena->stats.retained -= size;
+ if (config_prof)
+ extent_gprof_add(tsdn, extent);
+ }
+ if (!config_munmap && extent == NULL) {
+ extent = extent_grow_retained(tsdn, arena, r_extent_hooks,
+ new_addr, usize, pad, alignment, zero, commit, slab);
}
return (extent);
@@ -909,6 +1062,8 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
if (config_stats)
arena->stats.retained += extent_size_get(extent);
+ if (config_prof)
+ extent_gprof_sub(tsdn, extent);
extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained,
false, extent);