Add a per-arena oversize_threshold.

This can let manual arenas trade off memory and CPU the way auto arenas do.
This commit is contained in:
David Goldblatt 2020-11-11 13:34:43 -08:00 committed by David Goldblatt
parent 4ca3d91e96
commit cf2549a149
10 changed files with 194 additions and 12 deletions

View File

@ -229,6 +229,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/mq.c \ $(srcroot)test/unit/mq.c \
$(srcroot)test/unit/mtx.c \ $(srcroot)test/unit/mtx.c \
$(srcroot)test/unit/nstime.c \ $(srcroot)test/unit/nstime.c \
$(srcroot)test/unit/oversize_threshold.c \
$(srcroot)test/unit/pa.c \ $(srcroot)test/unit/pa.c \
$(srcroot)test/unit/pack.c \ $(srcroot)test/unit/pack.c \
$(srcroot)test/unit/pages.c \ $(srcroot)test/unit/pages.c \

View File

@ -123,7 +123,8 @@ pa_shard_ehooks_get(pa_shard_t *shard) {
/* Returns true on error. */ /* Returns true on error. */
bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx, unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
nstime_t *cur_time, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms); nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
ssize_t muzzy_decay_ms);
/* /*
* This isn't exposed to users; we allow late enablement of the HPA shard so * This isn't exposed to users; we allow late enablement of the HPA shard so

View File

@ -98,6 +98,9 @@ struct pac_s {
exp_grow_t exp_grow; exp_grow_t exp_grow;
malloc_mutex_t grow_mtx; malloc_mutex_t grow_mtx;
/* How large extents should be before getting auto-purged. */
atomic_zu_t oversize_threshold;
/* /*
* Decay-based purging state, responsible for scheduling extent state * Decay-based purging state, responsible for scheduling extent state
* transitions. * transitions.
@ -115,8 +118,9 @@ struct pac_s {
}; };
bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms, edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold,
ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx); ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
malloc_mutex_t *stats_mtx);
bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
size_t *new_limit); size_t *new_limit);
void pac_stats_merge(tsdn_t *tsdn, pac_t *pac, pac_stats_t *pac_stats_out, void pac_stats_merge(tsdn_t *tsdn, pac_t *pac, pac_stats_t *pac_stats_out,

View File

@ -1500,7 +1500,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
nstime_init_update(&cur_time); nstime_init_update(&cur_time);
if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind, if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind,
&arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx), &arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx),
&cur_time, arena_dirty_decay_ms_default_get(), &cur_time, oversize_threshold, arena_dirty_decay_ms_default_get(),
arena_muzzy_decay_ms_default_get())) { arena_muzzy_decay_ms_default_get())) {
goto label_error; goto label_error;
} }

View File

@ -151,6 +151,7 @@ CTL_PROTO(arena_i_purge)
CTL_PROTO(arena_i_reset) CTL_PROTO(arena_i_reset)
CTL_PROTO(arena_i_destroy) CTL_PROTO(arena_i_destroy)
CTL_PROTO(arena_i_dss) CTL_PROTO(arena_i_dss)
CTL_PROTO(arena_i_oversize_threshold)
CTL_PROTO(arena_i_dirty_decay_ms) CTL_PROTO(arena_i_dirty_decay_ms)
CTL_PROTO(arena_i_muzzy_decay_ms) CTL_PROTO(arena_i_muzzy_decay_ms)
CTL_PROTO(arena_i_extent_hooks) CTL_PROTO(arena_i_extent_hooks)
@ -431,6 +432,11 @@ static const ctl_named_node_t arena_i_node[] = {
{NAME("reset"), CTL(arena_i_reset)}, {NAME("reset"), CTL(arena_i_reset)},
{NAME("destroy"), CTL(arena_i_destroy)}, {NAME("destroy"), CTL(arena_i_destroy)},
{NAME("dss"), CTL(arena_i_dss)}, {NAME("dss"), CTL(arena_i_dss)},
/*
* Undocumented for now, since we anticipate an arena API in flux after
* we cut the last 5-series release.
*/
{NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)},
{NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},
{NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},
{NAME("extent_hooks"), CTL(arena_i_extent_hooks)}, {NAME("extent_hooks"), CTL(arena_i_extent_hooks)},
@ -2530,6 +2536,38 @@ label_return:
return ret; return ret;
} }
static int
arena_i_oversize_threshold_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
MIB_UNSIGNED(arena_ind, 1);
arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
if (arena == NULL) {
ret = EFAULT;
goto label_return;
}
if (oldp != NULL && oldlenp != NULL) {
size_t oldval = atomic_load_zu(
&arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
READ(oldval, size_t);
}
if (newp != NULL) {
if (newlen != sizeof(size_t)) {
ret = EINVAL;
goto label_return;
}
atomic_store_zu(&arena->pa_shard.pac.oversize_threshold,
*(size_t *)newp, ATOMIC_RELAXED);
}
ret = 0;
label_return:
return ret;
}
static int static int
arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {

View File

@ -983,8 +983,9 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
edata = extent_try_coalesce_large(tsdn, pac, ehooks, edata = extent_try_coalesce_large(tsdn, pac, ehooks,
ecache, edata, &coalesced, growing_retained); ecache, edata, &coalesced, growing_retained);
} while (coalesced); } while (coalesced);
if (edata_size_get(edata) >= oversize_threshold && if (edata_size_get(edata) >=
extent_may_force_decay(pac)) { atomic_load_zu(&pac->oversize_threshold, ATOMIC_RELAXED)
&& extent_may_force_decay(pac)) {
/* Shortcut to purge the oversize extent eagerly. */ /* Shortcut to purge the oversize extent eagerly. */
malloc_mutex_unlock(tsdn, &ecache->mtx); malloc_mutex_unlock(tsdn, &ecache->mtx);
extent_maximally_purge(tsdn, pac, ehooks, edata); extent_maximally_purge(tsdn, pac, ehooks, edata);

View File

@ -17,7 +17,8 @@ pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) {
bool bool
pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx, unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
nstime_t *cur_time, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) { nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
ssize_t muzzy_decay_ms) {
/* This will change eventually, but for now it should hold. */ /* This will change eventually, but for now it should hold. */
assert(base_ind_get(base) == ind); assert(base_ind_get(base) == ind);
if (edata_cache_init(&shard->edata_cache, base)) { if (edata_cache_init(&shard->edata_cache, base)) {
@ -25,8 +26,8 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
} }
if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache, if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache,
cur_time, dirty_decay_ms, muzzy_decay_ms, &stats->pac_stats, cur_time, oversize_threshold, dirty_decay_ms, muzzy_decay_ms,
stats_mtx)) { &stats->pac_stats, stats_mtx)) {
return true; return true;
} }

View File

@ -37,8 +37,9 @@ pac_decay_data_get(pac_t *pac, extent_state_t state,
bool bool
pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms, edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold,
ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx) { ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
malloc_mutex_t *stats_mtx) {
unsigned ind = base_ind_get(base); unsigned ind = base_ind_get(base);
/* /*
* Delay coalescing for dirty extents despite the disruptive effect on * Delay coalescing for dirty extents despite the disruptive effect on
@ -73,6 +74,8 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
return true; return true;
} }
atomic_store_zu(&pac->oversize_threshold, oversize_threshold,
ATOMIC_RELAXED);
if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) { if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) {
return true; return true;
} }

View File

@ -0,0 +1,131 @@
#include "test/jemalloc_test.h"
#include "jemalloc/internal/ctl.h"
static void
arena_mallctl(const char *mallctl_str, unsigned arena, void *oldp,
size_t *oldlen, void *newp, size_t newlen) {
int err;
char buf[100];
malloc_snprintf(buf, sizeof(buf), mallctl_str, arena);
err = mallctl(buf, oldp, oldlen, newp, newlen);
expect_d_eq(0, err, "Mallctl failed; %s", buf);
}
TEST_BEGIN(test_oversize_threshold_get_set) {
int err;
size_t old_threshold;
size_t new_threshold;
size_t threshold_sz = sizeof(old_threshold);
unsigned arena;
size_t arena_sz = sizeof(arena);
err = mallctl("arenas.create", (void *)&arena, &arena_sz, NULL, 0);
expect_d_eq(0, err, "Arena creation failed");
/* Just a write. */
new_threshold = 1024 * 1024;
arena_mallctl("arena.%u.oversize_threshold", arena, NULL, NULL,
&new_threshold, threshold_sz);
/* Read and write */
new_threshold = 2 * 1024 * 1024;
arena_mallctl("arena.%u.oversize_threshold", arena, &old_threshold,
&threshold_sz, &new_threshold, threshold_sz);
expect_zu_eq(1024 * 1024, old_threshold, "Should have read old value");
/* Just a read */
arena_mallctl("arena.%u.oversize_threshold", arena, &old_threshold,
&threshold_sz, NULL, 0);
expect_zu_eq(2 * 1024 * 1024, old_threshold, "Should have read old value");
}
TEST_END
static size_t max_purged = 0;
static bool
purge_forced_record_max(extent_hooks_t* hooks, void *addr, size_t sz,
size_t offset, size_t length, unsigned arena_ind) {
if (length > max_purged) {
max_purged = length;
}
return false;
}
static bool
dalloc_record_max(extent_hooks_t *extent_hooks, void *addr, size_t sz,
bool comitted, unsigned arena_ind) {
if (sz > max_purged) {
max_purged = sz;
}
return false;
}
extent_hooks_t max_recording_extent_hooks;
TEST_BEGIN(test_oversize_threshold) {
max_recording_extent_hooks = ehooks_default_extent_hooks;
max_recording_extent_hooks.purge_forced = &purge_forced_record_max;
max_recording_extent_hooks.dalloc = &dalloc_record_max;
extent_hooks_t *extent_hooks = &max_recording_extent_hooks;
int err;
unsigned arena;
size_t arena_sz = sizeof(arena);
err = mallctl("arenas.create", (void *)&arena, &arena_sz, NULL, 0);
expect_d_eq(0, err, "Arena creation failed");
arena_mallctl("arena.%u.extent_hooks", arena, NULL, NULL, &extent_hooks,
sizeof(extent_hooks));
/*
* This test will fundamentally race with purging, since we're going to
* check the dirty stats to see if our oversized allocation got purged.
* We don't want other purging to happen accidentally. We can't just
* disable purging entirely, though, since that will also disable
* oversize purging. Just set purging intervals to be very large.
*/
ssize_t decay_ms = 100 * 1000;
ssize_t decay_ms_sz = sizeof(decay_ms);
arena_mallctl("arena.%u.dirty_decay_ms", arena, NULL, NULL, &decay_ms,
decay_ms_sz);
arena_mallctl("arena.%u.muzzy_decay_ms", arena, NULL, NULL, &decay_ms,
decay_ms_sz);
/* Clean everything out. */
arena_mallctl("arena.%u.purge", arena, NULL, NULL, NULL, 0);
max_purged = 0;
/* Set threshold to 1MB. */
size_t threshold = 1024 * 1024;
size_t threshold_sz = sizeof(threshold);
arena_mallctl("arena.%u.oversize_threshold", arena, NULL, NULL,
&threshold, threshold_sz);
/* Allocating and freeing half a megabyte should leave them dirty. */
void *ptr = mallocx(512 * 1024, MALLOCX_ARENA(arena));
dallocx(ptr, MALLOCX_TCACHE_NONE);
expect_zu_lt(max_purged, 512 * 1024, "Expected no 512k purge");
/* Purge again to reset everything out. */
arena_mallctl("arena.%u.purge", arena, NULL, NULL, NULL, 0);
max_purged = 0;
/*
* Allocating and freeing 2 megabytes should leave them dirty because of
* the oversize threshold.
*/
ptr = mallocx(2 * 1024 * 1024, MALLOCX_ARENA(arena));
dallocx(ptr, MALLOCX_TCACHE_NONE);
expect_zu_ge(max_purged, 2 * 1024 * 1024, "Expected a 2MB purge");
}
TEST_END
int
main(void) {
return test_no_reentrancy(
test_oversize_threshold_get_set,
test_oversize_threshold);
}

View File

@ -63,9 +63,11 @@ test_data_t *init_test_data(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
nstime_t time; nstime_t time;
nstime_init(&time, 0); nstime_init(&time, 0);
const size_t oversize_threshold = 8 * 1024 * 1024;
err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap, err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap,
test_data->base, /* ind */ 1, &test_data->stats, test_data->base, /* ind */ 1, &test_data->stats,
&test_data->stats_mtx, &time, dirty_decay_ms, muzzy_decay_ms); &test_data->stats_mtx, &time, oversize_threshold, dirty_decay_ms,
muzzy_decay_ms);
assert_false(err, ""); assert_false(err, "");
return test_data; return test_data;