diff --git a/Makefile.in b/Makefile.in index ca9b17b3..03dbbdf5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -229,6 +229,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/mq.c \ $(srcroot)test/unit/mtx.c \ $(srcroot)test/unit/nstime.c \ + $(srcroot)test/unit/oversize_threshold.c \ $(srcroot)test/unit/pa.c \ $(srcroot)test/unit/pack.c \ $(srcroot)test/unit/pages.c \ diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 5e97d0b0..f1823e6b 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -123,7 +123,8 @@ pa_shard_ehooks_get(pa_shard_t *shard) { /* Returns true on error. */ bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx, - nstime_t *cur_time, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms); + nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms, + ssize_t muzzy_decay_ms); /* * This isn't exposed to users; we allow late enablement of the HPA shard so diff --git a/include/jemalloc/internal/pac.h b/include/jemalloc/internal/pac.h index b998b69a..6d4dfbaf 100644 --- a/include/jemalloc/internal/pac.h +++ b/include/jemalloc/internal/pac.h @@ -98,6 +98,9 @@ struct pac_s { exp_grow_t exp_grow; malloc_mutex_t grow_mtx; + /* How large extents should be before getting auto-purged. */ + atomic_zu_t oversize_threshold; + /* * Decay-based purging state, responsible for scheduling extent state * transitions. @@ -115,8 +118,9 @@ struct pac_s { }; bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, - edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms, - ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx); + edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold, + ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, + malloc_mutex_t *stats_mtx); bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit); void pac_stats_merge(tsdn_t *tsdn, pac_t *pac, pac_stats_t *pac_stats_out, diff --git a/src/arena.c b/src/arena.c index 360827ef..7099713a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1500,7 +1500,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { nstime_init_update(&cur_time); if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind, &arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx), - &cur_time, arena_dirty_decay_ms_default_get(), + &cur_time, oversize_threshold, arena_dirty_decay_ms_default_get(), arena_muzzy_decay_ms_default_get())) { goto label_error; } diff --git a/src/ctl.c b/src/ctl.c index d5dd1d16..4bb422a2 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -151,6 +151,7 @@ CTL_PROTO(arena_i_purge) CTL_PROTO(arena_i_reset) CTL_PROTO(arena_i_destroy) CTL_PROTO(arena_i_dss) +CTL_PROTO(arena_i_oversize_threshold) CTL_PROTO(arena_i_dirty_decay_ms) CTL_PROTO(arena_i_muzzy_decay_ms) CTL_PROTO(arena_i_extent_hooks) @@ -431,6 +432,11 @@ static const ctl_named_node_t arena_i_node[] = { {NAME("reset"), CTL(arena_i_reset)}, {NAME("destroy"), CTL(arena_i_destroy)}, {NAME("dss"), CTL(arena_i_dss)}, + /* + * Undocumented for now, since we anticipate an arena API in flux after + * we cut the last 5-series release. + */ + {NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)}, {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}, @@ -2530,6 +2536,38 @@ label_return: return ret; } +static int +arena_i_oversize_threshold_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + unsigned arena_ind; + MIB_UNSIGNED(arena_ind, 1); + + arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); + if (arena == NULL) { + ret = EFAULT; + goto label_return; + } + + if (oldp != NULL && oldlenp != NULL) { + size_t oldval = atomic_load_zu( + &arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED); + READ(oldval, size_t); + } + if (newp != NULL) { + if (newlen != sizeof(size_t)) { + ret = EINVAL; + goto label_return; + } + atomic_store_zu(&arena->pa_shard.pac.oversize_threshold, + *(size_t *)newp, ATOMIC_RELAXED); + } + ret = 0; +label_return: + return ret; +} + static int arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { diff --git a/src/extent.c b/src/extent.c index c7dcc2e9..378bc733 100644 --- a/src/extent.c +++ b/src/extent.c @@ -983,8 +983,9 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata = extent_try_coalesce_large(tsdn, pac, ehooks, ecache, edata, &coalesced, growing_retained); } while (coalesced); - if (edata_size_get(edata) >= oversize_threshold && - extent_may_force_decay(pac)) { + if (edata_size_get(edata) >= + atomic_load_zu(&pac->oversize_threshold, ATOMIC_RELAXED) + && extent_may_force_decay(pac)) { /* Shortcut to purge the oversize extent eagerly. */ malloc_mutex_unlock(tsdn, &ecache->mtx); extent_maximally_purge(tsdn, pac, ehooks, edata); diff --git a/src/pa.c b/src/pa.c index aee7bcd8..e5fcbb7b 100644 --- a/src/pa.c +++ b/src/pa.c @@ -17,7 +17,8 @@ pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) { bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx, - nstime_t *cur_time, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) { + nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms, + ssize_t muzzy_decay_ms) { /* This will change eventually, but for now it should hold. */ assert(base_ind_get(base) == ind); if (edata_cache_init(&shard->edata_cache, base)) { @@ -25,8 +26,8 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, } if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache, - cur_time, dirty_decay_ms, muzzy_decay_ms, &stats->pac_stats, - stats_mtx)) { + cur_time, oversize_threshold, dirty_decay_ms, muzzy_decay_ms, + &stats->pac_stats, stats_mtx)) { return true; } diff --git a/src/pac.c b/src/pac.c index 07c9d23d..80646155 100644 --- a/src/pac.c +++ b/src/pac.c @@ -37,8 +37,9 @@ pac_decay_data_get(pac_t *pac, extent_state_t state, bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, - edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms, - ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx) { + edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold, + ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, + malloc_mutex_t *stats_mtx) { unsigned ind = base_ind_get(base); /* * Delay coalescing for dirty extents despite the disruptive effect on @@ -73,6 +74,8 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { return true; } + atomic_store_zu(&pac->oversize_threshold, oversize_threshold, + ATOMIC_RELAXED); if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) { return true; } diff --git a/test/unit/oversize_threshold.c b/test/unit/oversize_threshold.c new file mode 100644 index 00000000..e374b142 --- /dev/null +++ b/test/unit/oversize_threshold.c @@ -0,0 +1,131 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/ctl.h" + +static void +arena_mallctl(const char *mallctl_str, unsigned arena, void *oldp, + size_t *oldlen, void *newp, size_t newlen) { + int err; + char buf[100]; + malloc_snprintf(buf, sizeof(buf), mallctl_str, arena); + + err = mallctl(buf, oldp, oldlen, newp, newlen); + expect_d_eq(0, err, "Mallctl failed; %s", buf); +} + +TEST_BEGIN(test_oversize_threshold_get_set) { + int err; + size_t old_threshold; + size_t new_threshold; + size_t threshold_sz = sizeof(old_threshold); + + unsigned arena; + size_t arena_sz = sizeof(arena); + err = mallctl("arenas.create", (void *)&arena, &arena_sz, NULL, 0); + expect_d_eq(0, err, "Arena creation failed"); + + /* Just a write. */ + new_threshold = 1024 * 1024; + arena_mallctl("arena.%u.oversize_threshold", arena, NULL, NULL, + &new_threshold, threshold_sz); + + /* Read and write */ + new_threshold = 2 * 1024 * 1024; + arena_mallctl("arena.%u.oversize_threshold", arena, &old_threshold, + &threshold_sz, &new_threshold, threshold_sz); + expect_zu_eq(1024 * 1024, old_threshold, "Should have read old value"); + + /* Just a read */ + arena_mallctl("arena.%u.oversize_threshold", arena, &old_threshold, + &threshold_sz, NULL, 0); + expect_zu_eq(2 * 1024 * 1024, old_threshold, "Should have read old value"); +} +TEST_END + +static size_t max_purged = 0; +static bool +purge_forced_record_max(extent_hooks_t* hooks, void *addr, size_t sz, + size_t offset, size_t length, unsigned arena_ind) { + if (length > max_purged) { + max_purged = length; + } + return false; +} + +static bool +dalloc_record_max(extent_hooks_t *extent_hooks, void *addr, size_t sz, + bool comitted, unsigned arena_ind) { + if (sz > max_purged) { + max_purged = sz; + } + return false; +} + +extent_hooks_t max_recording_extent_hooks; + +TEST_BEGIN(test_oversize_threshold) { + max_recording_extent_hooks = ehooks_default_extent_hooks; + max_recording_extent_hooks.purge_forced = &purge_forced_record_max; + max_recording_extent_hooks.dalloc = &dalloc_record_max; + + extent_hooks_t *extent_hooks = &max_recording_extent_hooks; + + int err; + + unsigned arena; + size_t arena_sz = sizeof(arena); + err = mallctl("arenas.create", (void *)&arena, &arena_sz, NULL, 0); + expect_d_eq(0, err, "Arena creation failed"); + arena_mallctl("arena.%u.extent_hooks", arena, NULL, NULL, &extent_hooks, + sizeof(extent_hooks)); + + /* + * This test will fundamentally race with purging, since we're going to + * check the dirty stats to see if our oversized allocation got purged. + * We don't want other purging to happen accidentally. We can't just + * disable purging entirely, though, since that will also disable + * oversize purging. Just set purging intervals to be very large. + */ + ssize_t decay_ms = 100 * 1000; + ssize_t decay_ms_sz = sizeof(decay_ms); + arena_mallctl("arena.%u.dirty_decay_ms", arena, NULL, NULL, &decay_ms, + decay_ms_sz); + arena_mallctl("arena.%u.muzzy_decay_ms", arena, NULL, NULL, &decay_ms, + decay_ms_sz); + + /* Clean everything out. */ + arena_mallctl("arena.%u.purge", arena, NULL, NULL, NULL, 0); + max_purged = 0; + + /* Set threshold to 1MB. */ + size_t threshold = 1024 * 1024; + size_t threshold_sz = sizeof(threshold); + arena_mallctl("arena.%u.oversize_threshold", arena, NULL, NULL, + &threshold, threshold_sz); + + /* Allocating and freeing half a megabyte should leave them dirty. */ + void *ptr = mallocx(512 * 1024, MALLOCX_ARENA(arena)); + dallocx(ptr, MALLOCX_TCACHE_NONE); + expect_zu_lt(max_purged, 512 * 1024, "Expected no 512k purge"); + + /* Purge again to reset everything out. */ + arena_mallctl("arena.%u.purge", arena, NULL, NULL, NULL, 0); + max_purged = 0; + + /* + * Allocating and freeing 2 megabytes should leave them dirty because of + * the oversize threshold. + */ + ptr = mallocx(2 * 1024 * 1024, MALLOCX_ARENA(arena)); + dallocx(ptr, MALLOCX_TCACHE_NONE); + expect_zu_ge(max_purged, 2 * 1024 * 1024, "Expected a 2MB purge"); +} +TEST_END + +int +main(void) { + return test_no_reentrancy( + test_oversize_threshold_get_set, + test_oversize_threshold); +} + diff --git a/test/unit/pa.c b/test/unit/pa.c index 3a910235..dacd8e70 100644 --- a/test/unit/pa.c +++ b/test/unit/pa.c @@ -63,9 +63,11 @@ test_data_t *init_test_data(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) { nstime_t time; nstime_init(&time, 0); + const size_t oversize_threshold = 8 * 1024 * 1024; err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap, test_data->base, /* ind */ 1, &test_data->stats, - &test_data->stats_mtx, &time, dirty_decay_ms, muzzy_decay_ms); + &test_data->stats_mtx, &time, oversize_threshold, dirty_decay_ms, + muzzy_decay_ms); assert_false(err, ""); return test_data;