diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index 8151b5ba..895b2d4d 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -1683,6 +1683,22 @@ malloc_conf = "xmalloc:true";]]>
for additional information.
+
+
+ arena.<i>.retain_grow_limit
+ (size_t)
+ rw
+
+ Maximum size to grow retained region (only relevant when
+ opt.retain is
+ enabled). This controls the maximum increment to expand virtual memory,
+ or allocation through arena.<i>extent_hooks.
+ In particular, if customized extent hooks reserve physical memory
+ (e.g. 1G huge pages), this is useful to control the allocation hook's
+ input size. The default is no limit.
+
+
arena.<i>.extent_hooks
diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h
index 4e546c3b..5a0e3add 100644
--- a/include/jemalloc/internal/arena_externs.h
+++ b/include/jemalloc/internal/arena_externs.h
@@ -77,6 +77,8 @@ ssize_t arena_dirty_decay_ms_default_get(void);
bool arena_dirty_decay_ms_default_set(ssize_t decay_ms);
ssize_t arena_muzzy_decay_ms_default_get(void);
bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
+bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena,
+ size_t *old_limit, size_t *new_limit);
unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal);
diff --git a/include/jemalloc/internal/arena_structs_b.h b/include/jemalloc/internal/arena_structs_b.h
index c4e4310d..f74ea97d 100644
--- a/include/jemalloc/internal/arena_structs_b.h
+++ b/include/jemalloc/internal/arena_structs_b.h
@@ -240,9 +240,14 @@ struct arena_s {
* be effective even if multiple arenas' extent allocation requests are
* highly interleaved.
*
+ * retain_grow_limit is the max allowed size ind to expand (unless the
+ * required size is greater). Default is no limit, and controlled
+ * through mallctl only.
+ *
* Synchronization: extent_grow_mtx
*/
pszind_t extent_grow_next;
+ pszind_t retain_grow_limit;
malloc_mutex_t extent_grow_mtx;
/*
diff --git a/include/jemalloc/internal/extent_types.h b/include/jemalloc/internal/extent_types.h
index b6905ce1..7efcd3a4 100644
--- a/include/jemalloc/internal/extent_types.h
+++ b/include/jemalloc/internal/extent_types.h
@@ -6,4 +6,6 @@ typedef struct extents_s extents_t;
#define EXTENT_HOOKS_INITIALIZER NULL
+#define EXTENT_GROW_MAX_PIND (NPSIZES - 1)
+
#endif /* JEMALLOC_INTERNAL_EXTENT_TYPES_H */
diff --git a/src/arena.c b/src/arena.c
index 43ba6018..91dce1f5 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -1886,6 +1886,33 @@ arena_muzzy_decay_ms_default_set(ssize_t decay_ms) {
return false;
}
+bool
+arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit,
+ size_t *new_limit) {
+ assert(opt_retain);
+
+ pszind_t new_ind JEMALLOC_CC_SILENCE_INIT(0);
+ if (new_limit != NULL) {
+ size_t limit = *new_limit;
+ /* Grow no more than the new limit. */
+ if ((new_ind = sz_psz2ind(limit + 1) - 1) >
+ EXTENT_GROW_MAX_PIND) {
+ return true;
+ }
+ }
+
+ malloc_mutex_lock(tsd_tsdn(tsd), &arena->extent_grow_mtx);
+ if (old_limit != NULL) {
+ *old_limit = sz_pind2sz(arena->retain_grow_limit);
+ }
+ if (new_limit != NULL) {
+ arena->retain_grow_limit = new_ind;
+ }
+ malloc_mutex_unlock(tsd_tsdn(tsd), &arena->extent_grow_mtx);
+
+ return false;
+}
+
unsigned
arena_nthreads_get(arena_t *arena, bool internal) {
return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED);
@@ -2013,6 +2040,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
}
arena->extent_grow_next = sz_psz2ind(HUGEPAGE);
+ arena->retain_grow_limit = EXTENT_GROW_MAX_PIND;
if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow",
WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
goto label_error;
diff --git a/src/ctl.c b/src/ctl.c
index a2f3837a..11cd68db 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -118,6 +118,7 @@ CTL_PROTO(arena_i_dss)
CTL_PROTO(arena_i_dirty_decay_ms)
CTL_PROTO(arena_i_muzzy_decay_ms)
CTL_PROTO(arena_i_extent_hooks)
+CTL_PROTO(arena_i_retain_grow_limit)
INDEX_PROTO(arena_i)
CTL_PROTO(arenas_bin_i_size)
CTL_PROTO(arenas_bin_i_nregs)
@@ -320,7 +321,8 @@ static const ctl_named_node_t arena_i_node[] = {
{NAME("dss"), CTL(arena_i_dss)},
{NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},
{NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},
- {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}
+ {NAME("extent_hooks"), CTL(arena_i_extent_hooks)},
+ {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)}
};
static const ctl_named_node_t super_arena_i_node[] = {
{NAME(""), CHILD(named, arena_i)}
@@ -2199,6 +2201,42 @@ label_return:
return ret;
}
+static int
+arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
+ int ret;
+ unsigned arena_ind;
+ arena_t *arena;
+
+ if (!opt_retain) {
+ /* Only relevant when retain is enabled. */
+ return ENOENT;
+ }
+
+ malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
+ MIB_UNSIGNED(arena_ind, 1);
+ if (arena_ind < narenas_total_get() && (arena =
+ arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
+ size_t old_limit, new_limit;
+ if (newp != NULL) {
+ WRITE(new_limit, size_t);
+ }
+ bool err = arena_retain_grow_limit_get_set(tsd, arena,
+ &old_limit, newp != NULL ? &new_limit : NULL);
+ if (!err) {
+ READ(old_limit, size_t);
+ ret = 0;
+ } else {
+ ret = EFAULT;
+ }
+ } else {
+ ret = EFAULT;
+ }
+label_return:
+ malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
+ return ret;
+}
+
static const ctl_named_node_t *
arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
const ctl_named_node_t *ret;
@@ -2260,7 +2298,7 @@ arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
ret = EINVAL;
goto label_return;
}
- if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp)
+ if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp)
: arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) {
ret = EFAULT;
goto label_return;
diff --git a/src/extent.c b/src/extent.c
index 497f4e40..d1324f99 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -1284,13 +1284,14 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
}
/*
- * Increment extent_grow_next if doing so wouldn't exceed the legal
+ * Increment extent_grow_next if doing so wouldn't exceed the allowed
* range.
*/
- if (arena->extent_grow_next + egn_skip + 1 < NPSIZES) {
+ if (arena->extent_grow_next + egn_skip + 1 <=
+ arena->retain_grow_limit) {
arena->extent_grow_next += egn_skip + 1;
} else {
- arena->extent_grow_next = NPSIZES - 1;
+ arena->extent_grow_next = arena->retain_grow_limit;
}
/* All opportunities for failure are past. */
malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c
index 5612cce5..94f801e3 100644
--- a/test/unit/mallctl.c
+++ b/test/unit/mallctl.c
@@ -556,6 +556,54 @@ TEST_BEGIN(test_arena_i_dss) {
}
TEST_END
+TEST_BEGIN(test_arena_i_retain_grow_limit) {
+ size_t old_limit, new_limit, default_limit;
+ size_t mib[3];
+ size_t miblen;
+
+ bool retain_enabled;
+ size_t sz = sizeof(retain_enabled);
+ assert_d_eq(mallctl("opt.retain", &retain_enabled, &sz, NULL, 0),
+ 0, "Unexpected mallctl() failure");
+ test_skip_if(!retain_enabled);
+
+ sz = sizeof(default_limit);
+ miblen = sizeof(mib)/sizeof(size_t);
+ assert_d_eq(mallctlnametomib("arena.0.retain_grow_limit", mib, &miblen),
+ 0, "Unexpected mallctlnametomib() error");
+
+ assert_d_eq(mallctlbymib(mib, miblen, &default_limit, &sz, NULL, 0), 0,
+ "Unexpected mallctl() failure");
+ assert_zu_eq(default_limit, sz_pind2sz(EXTENT_GROW_MAX_PIND),
+ "Unexpected default for retain_grow_limit");
+
+ new_limit = PAGE - 1;
+ assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &new_limit,
+ sizeof(new_limit)), EFAULT, "Unexpected mallctl() success");
+
+ new_limit = PAGE + 1;
+ assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &new_limit,
+ sizeof(new_limit)), 0, "Unexpected mallctl() failure");
+ assert_d_eq(mallctlbymib(mib, miblen, &old_limit, &sz, NULL, 0), 0,
+ "Unexpected mallctl() failure");
+ assert_zu_eq(old_limit, PAGE,
+ "Unexpected value for retain_grow_limit");
+
+ /* Expect grow less than psize class 10. */
+ new_limit = sz_pind2sz(10) - 1;
+ assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &new_limit,
+ sizeof(new_limit)), 0, "Unexpected mallctl() failure");
+ assert_d_eq(mallctlbymib(mib, miblen, &old_limit, &sz, NULL, 0), 0,
+ "Unexpected mallctl() failure");
+ assert_zu_eq(old_limit, sz_pind2sz(9),
+ "Unexpected value for retain_grow_limit");
+
+ /* Restore to default. */
+ assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &default_limit,
+ sizeof(default_limit)), 0, "Unexpected mallctl() failure");
+}
+TEST_END
+
TEST_BEGIN(test_arenas_dirty_decay_ms) {
ssize_t dirty_decay_ms, orig_dirty_decay_ms, prev_dirty_decay_ms;
size_t sz = sizeof(ssize_t);
@@ -727,6 +775,7 @@ main(void) {
test_arena_i_purge,
test_arena_i_decay,
test_arena_i_dss,
+ test_arena_i_retain_grow_limit,
test_arenas_dirty_decay_ms,
test_arenas_muzzy_decay_ms,
test_arenas_constants,