Add a per-arena oversize_threshold.
This can let manual arenas trade off memory and CPU the way auto arenas do.
This commit is contained in:
parent
4ca3d91e96
commit
cf2549a149
@ -229,6 +229,7 @@ TESTS_UNIT := \
|
|||||||
$(srcroot)test/unit/mq.c \
|
$(srcroot)test/unit/mq.c \
|
||||||
$(srcroot)test/unit/mtx.c \
|
$(srcroot)test/unit/mtx.c \
|
||||||
$(srcroot)test/unit/nstime.c \
|
$(srcroot)test/unit/nstime.c \
|
||||||
|
$(srcroot)test/unit/oversize_threshold.c \
|
||||||
$(srcroot)test/unit/pa.c \
|
$(srcroot)test/unit/pa.c \
|
||||||
$(srcroot)test/unit/pack.c \
|
$(srcroot)test/unit/pack.c \
|
||||||
$(srcroot)test/unit/pages.c \
|
$(srcroot)test/unit/pages.c \
|
||||||
|
@ -123,7 +123,8 @@ pa_shard_ehooks_get(pa_shard_t *shard) {
|
|||||||
/* Returns true on error. */
|
/* Returns true on error. */
|
||||||
bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
||||||
unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
|
unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
|
||||||
nstime_t *cur_time, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms);
|
nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
|
||||||
|
ssize_t muzzy_decay_ms);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* This isn't exposed to users; we allow late enablement of the HPA shard so
|
* This isn't exposed to users; we allow late enablement of the HPA shard so
|
||||||
|
@ -98,6 +98,9 @@ struct pac_s {
|
|||||||
exp_grow_t exp_grow;
|
exp_grow_t exp_grow;
|
||||||
malloc_mutex_t grow_mtx;
|
malloc_mutex_t grow_mtx;
|
||||||
|
|
||||||
|
/* How large extents should be before getting auto-purged. */
|
||||||
|
atomic_zu_t oversize_threshold;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Decay-based purging state, responsible for scheduling extent state
|
* Decay-based purging state, responsible for scheduling extent state
|
||||||
* transitions.
|
* transitions.
|
||||||
@ -115,8 +118,9 @@ struct pac_s {
|
|||||||
};
|
};
|
||||||
|
|
||||||
bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
|
bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
|
||||||
edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms,
|
edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold,
|
||||||
ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx);
|
ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
|
||||||
|
malloc_mutex_t *stats_mtx);
|
||||||
bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
|
bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
|
||||||
size_t *new_limit);
|
size_t *new_limit);
|
||||||
void pac_stats_merge(tsdn_t *tsdn, pac_t *pac, pac_stats_t *pac_stats_out,
|
void pac_stats_merge(tsdn_t *tsdn, pac_t *pac, pac_stats_t *pac_stats_out,
|
||||||
|
@ -1500,7 +1500,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
|
|||||||
nstime_init_update(&cur_time);
|
nstime_init_update(&cur_time);
|
||||||
if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind,
|
if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind,
|
||||||
&arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx),
|
&arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx),
|
||||||
&cur_time, arena_dirty_decay_ms_default_get(),
|
&cur_time, oversize_threshold, arena_dirty_decay_ms_default_get(),
|
||||||
arena_muzzy_decay_ms_default_get())) {
|
arena_muzzy_decay_ms_default_get())) {
|
||||||
goto label_error;
|
goto label_error;
|
||||||
}
|
}
|
||||||
|
38
src/ctl.c
38
src/ctl.c
@ -151,6 +151,7 @@ CTL_PROTO(arena_i_purge)
|
|||||||
CTL_PROTO(arena_i_reset)
|
CTL_PROTO(arena_i_reset)
|
||||||
CTL_PROTO(arena_i_destroy)
|
CTL_PROTO(arena_i_destroy)
|
||||||
CTL_PROTO(arena_i_dss)
|
CTL_PROTO(arena_i_dss)
|
||||||
|
CTL_PROTO(arena_i_oversize_threshold)
|
||||||
CTL_PROTO(arena_i_dirty_decay_ms)
|
CTL_PROTO(arena_i_dirty_decay_ms)
|
||||||
CTL_PROTO(arena_i_muzzy_decay_ms)
|
CTL_PROTO(arena_i_muzzy_decay_ms)
|
||||||
CTL_PROTO(arena_i_extent_hooks)
|
CTL_PROTO(arena_i_extent_hooks)
|
||||||
@ -431,6 +432,11 @@ static const ctl_named_node_t arena_i_node[] = {
|
|||||||
{NAME("reset"), CTL(arena_i_reset)},
|
{NAME("reset"), CTL(arena_i_reset)},
|
||||||
{NAME("destroy"), CTL(arena_i_destroy)},
|
{NAME("destroy"), CTL(arena_i_destroy)},
|
||||||
{NAME("dss"), CTL(arena_i_dss)},
|
{NAME("dss"), CTL(arena_i_dss)},
|
||||||
|
/*
|
||||||
|
* Undocumented for now, since we anticipate an arena API in flux after
|
||||||
|
* we cut the last 5-series release.
|
||||||
|
*/
|
||||||
|
{NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)},
|
||||||
{NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},
|
{NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},
|
||||||
{NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},
|
{NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},
|
||||||
{NAME("extent_hooks"), CTL(arena_i_extent_hooks)},
|
{NAME("extent_hooks"), CTL(arena_i_extent_hooks)},
|
||||||
@ -2530,6 +2536,38 @@ label_return:
|
|||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
arena_i_oversize_threshold_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
|
||||||
|
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
|
||||||
|
int ret;
|
||||||
|
|
||||||
|
unsigned arena_ind;
|
||||||
|
MIB_UNSIGNED(arena_ind, 1);
|
||||||
|
|
||||||
|
arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
|
||||||
|
if (arena == NULL) {
|
||||||
|
ret = EFAULT;
|
||||||
|
goto label_return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (oldp != NULL && oldlenp != NULL) {
|
||||||
|
size_t oldval = atomic_load_zu(
|
||||||
|
&arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
|
||||||
|
READ(oldval, size_t);
|
||||||
|
}
|
||||||
|
if (newp != NULL) {
|
||||||
|
if (newlen != sizeof(size_t)) {
|
||||||
|
ret = EINVAL;
|
||||||
|
goto label_return;
|
||||||
|
}
|
||||||
|
atomic_store_zu(&arena->pa_shard.pac.oversize_threshold,
|
||||||
|
*(size_t *)newp, ATOMIC_RELAXED);
|
||||||
|
}
|
||||||
|
ret = 0;
|
||||||
|
label_return:
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
|
arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
|
||||||
void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {
|
void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {
|
||||||
|
@ -983,8 +983,9 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
|
|||||||
edata = extent_try_coalesce_large(tsdn, pac, ehooks,
|
edata = extent_try_coalesce_large(tsdn, pac, ehooks,
|
||||||
ecache, edata, &coalesced, growing_retained);
|
ecache, edata, &coalesced, growing_retained);
|
||||||
} while (coalesced);
|
} while (coalesced);
|
||||||
if (edata_size_get(edata) >= oversize_threshold &&
|
if (edata_size_get(edata) >=
|
||||||
extent_may_force_decay(pac)) {
|
atomic_load_zu(&pac->oversize_threshold, ATOMIC_RELAXED)
|
||||||
|
&& extent_may_force_decay(pac)) {
|
||||||
/* Shortcut to purge the oversize extent eagerly. */
|
/* Shortcut to purge the oversize extent eagerly. */
|
||||||
malloc_mutex_unlock(tsdn, &ecache->mtx);
|
malloc_mutex_unlock(tsdn, &ecache->mtx);
|
||||||
extent_maximally_purge(tsdn, pac, ehooks, edata);
|
extent_maximally_purge(tsdn, pac, ehooks, edata);
|
||||||
|
7
src/pa.c
7
src/pa.c
@ -17,7 +17,8 @@ pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) {
|
|||||||
bool
|
bool
|
||||||
pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
||||||
unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
|
unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
|
||||||
nstime_t *cur_time, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
|
nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
|
||||||
|
ssize_t muzzy_decay_ms) {
|
||||||
/* This will change eventually, but for now it should hold. */
|
/* This will change eventually, but for now it should hold. */
|
||||||
assert(base_ind_get(base) == ind);
|
assert(base_ind_get(base) == ind);
|
||||||
if (edata_cache_init(&shard->edata_cache, base)) {
|
if (edata_cache_init(&shard->edata_cache, base)) {
|
||||||
@ -25,8 +26,8 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache,
|
if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache,
|
||||||
cur_time, dirty_decay_ms, muzzy_decay_ms, &stats->pac_stats,
|
cur_time, oversize_threshold, dirty_decay_ms, muzzy_decay_ms,
|
||||||
stats_mtx)) {
|
&stats->pac_stats, stats_mtx)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -37,8 +37,9 @@ pac_decay_data_get(pac_t *pac, extent_state_t state,
|
|||||||
|
|
||||||
bool
|
bool
|
||||||
pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
|
pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
|
||||||
edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms,
|
edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold,
|
||||||
ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx) {
|
ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
|
||||||
|
malloc_mutex_t *stats_mtx) {
|
||||||
unsigned ind = base_ind_get(base);
|
unsigned ind = base_ind_get(base);
|
||||||
/*
|
/*
|
||||||
* Delay coalescing for dirty extents despite the disruptive effect on
|
* Delay coalescing for dirty extents despite the disruptive effect on
|
||||||
@ -73,6 +74,8 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
|
|||||||
WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
|
WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
atomic_store_zu(&pac->oversize_threshold, oversize_threshold,
|
||||||
|
ATOMIC_RELAXED);
|
||||||
if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) {
|
if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
131
test/unit/oversize_threshold.c
Normal file
131
test/unit/oversize_threshold.c
Normal file
@ -0,0 +1,131 @@
|
|||||||
|
#include "test/jemalloc_test.h"
|
||||||
|
|
||||||
|
#include "jemalloc/internal/ctl.h"
|
||||||
|
|
||||||
|
static void
|
||||||
|
arena_mallctl(const char *mallctl_str, unsigned arena, void *oldp,
|
||||||
|
size_t *oldlen, void *newp, size_t newlen) {
|
||||||
|
int err;
|
||||||
|
char buf[100];
|
||||||
|
malloc_snprintf(buf, sizeof(buf), mallctl_str, arena);
|
||||||
|
|
||||||
|
err = mallctl(buf, oldp, oldlen, newp, newlen);
|
||||||
|
expect_d_eq(0, err, "Mallctl failed; %s", buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_BEGIN(test_oversize_threshold_get_set) {
|
||||||
|
int err;
|
||||||
|
size_t old_threshold;
|
||||||
|
size_t new_threshold;
|
||||||
|
size_t threshold_sz = sizeof(old_threshold);
|
||||||
|
|
||||||
|
unsigned arena;
|
||||||
|
size_t arena_sz = sizeof(arena);
|
||||||
|
err = mallctl("arenas.create", (void *)&arena, &arena_sz, NULL, 0);
|
||||||
|
expect_d_eq(0, err, "Arena creation failed");
|
||||||
|
|
||||||
|
/* Just a write. */
|
||||||
|
new_threshold = 1024 * 1024;
|
||||||
|
arena_mallctl("arena.%u.oversize_threshold", arena, NULL, NULL,
|
||||||
|
&new_threshold, threshold_sz);
|
||||||
|
|
||||||
|
/* Read and write */
|
||||||
|
new_threshold = 2 * 1024 * 1024;
|
||||||
|
arena_mallctl("arena.%u.oversize_threshold", arena, &old_threshold,
|
||||||
|
&threshold_sz, &new_threshold, threshold_sz);
|
||||||
|
expect_zu_eq(1024 * 1024, old_threshold, "Should have read old value");
|
||||||
|
|
||||||
|
/* Just a read */
|
||||||
|
arena_mallctl("arena.%u.oversize_threshold", arena, &old_threshold,
|
||||||
|
&threshold_sz, NULL, 0);
|
||||||
|
expect_zu_eq(2 * 1024 * 1024, old_threshold, "Should have read old value");
|
||||||
|
}
|
||||||
|
TEST_END
|
||||||
|
|
||||||
|
static size_t max_purged = 0;
|
||||||
|
static bool
|
||||||
|
purge_forced_record_max(extent_hooks_t* hooks, void *addr, size_t sz,
|
||||||
|
size_t offset, size_t length, unsigned arena_ind) {
|
||||||
|
if (length > max_purged) {
|
||||||
|
max_purged = length;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool
|
||||||
|
dalloc_record_max(extent_hooks_t *extent_hooks, void *addr, size_t sz,
|
||||||
|
bool comitted, unsigned arena_ind) {
|
||||||
|
if (sz > max_purged) {
|
||||||
|
max_purged = sz;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
extent_hooks_t max_recording_extent_hooks;
|
||||||
|
|
||||||
|
TEST_BEGIN(test_oversize_threshold) {
|
||||||
|
max_recording_extent_hooks = ehooks_default_extent_hooks;
|
||||||
|
max_recording_extent_hooks.purge_forced = &purge_forced_record_max;
|
||||||
|
max_recording_extent_hooks.dalloc = &dalloc_record_max;
|
||||||
|
|
||||||
|
extent_hooks_t *extent_hooks = &max_recording_extent_hooks;
|
||||||
|
|
||||||
|
int err;
|
||||||
|
|
||||||
|
unsigned arena;
|
||||||
|
size_t arena_sz = sizeof(arena);
|
||||||
|
err = mallctl("arenas.create", (void *)&arena, &arena_sz, NULL, 0);
|
||||||
|
expect_d_eq(0, err, "Arena creation failed");
|
||||||
|
arena_mallctl("arena.%u.extent_hooks", arena, NULL, NULL, &extent_hooks,
|
||||||
|
sizeof(extent_hooks));
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This test will fundamentally race with purging, since we're going to
|
||||||
|
* check the dirty stats to see if our oversized allocation got purged.
|
||||||
|
* We don't want other purging to happen accidentally. We can't just
|
||||||
|
* disable purging entirely, though, since that will also disable
|
||||||
|
* oversize purging. Just set purging intervals to be very large.
|
||||||
|
*/
|
||||||
|
ssize_t decay_ms = 100 * 1000;
|
||||||
|
ssize_t decay_ms_sz = sizeof(decay_ms);
|
||||||
|
arena_mallctl("arena.%u.dirty_decay_ms", arena, NULL, NULL, &decay_ms,
|
||||||
|
decay_ms_sz);
|
||||||
|
arena_mallctl("arena.%u.muzzy_decay_ms", arena, NULL, NULL, &decay_ms,
|
||||||
|
decay_ms_sz);
|
||||||
|
|
||||||
|
/* Clean everything out. */
|
||||||
|
arena_mallctl("arena.%u.purge", arena, NULL, NULL, NULL, 0);
|
||||||
|
max_purged = 0;
|
||||||
|
|
||||||
|
/* Set threshold to 1MB. */
|
||||||
|
size_t threshold = 1024 * 1024;
|
||||||
|
size_t threshold_sz = sizeof(threshold);
|
||||||
|
arena_mallctl("arena.%u.oversize_threshold", arena, NULL, NULL,
|
||||||
|
&threshold, threshold_sz);
|
||||||
|
|
||||||
|
/* Allocating and freeing half a megabyte should leave them dirty. */
|
||||||
|
void *ptr = mallocx(512 * 1024, MALLOCX_ARENA(arena));
|
||||||
|
dallocx(ptr, MALLOCX_TCACHE_NONE);
|
||||||
|
expect_zu_lt(max_purged, 512 * 1024, "Expected no 512k purge");
|
||||||
|
|
||||||
|
/* Purge again to reset everything out. */
|
||||||
|
arena_mallctl("arena.%u.purge", arena, NULL, NULL, NULL, 0);
|
||||||
|
max_purged = 0;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Allocating and freeing 2 megabytes should leave them dirty because of
|
||||||
|
* the oversize threshold.
|
||||||
|
*/
|
||||||
|
ptr = mallocx(2 * 1024 * 1024, MALLOCX_ARENA(arena));
|
||||||
|
dallocx(ptr, MALLOCX_TCACHE_NONE);
|
||||||
|
expect_zu_ge(max_purged, 2 * 1024 * 1024, "Expected a 2MB purge");
|
||||||
|
}
|
||||||
|
TEST_END
|
||||||
|
|
||||||
|
int
|
||||||
|
main(void) {
|
||||||
|
return test_no_reentrancy(
|
||||||
|
test_oversize_threshold_get_set,
|
||||||
|
test_oversize_threshold);
|
||||||
|
}
|
||||||
|
|
@ -63,9 +63,11 @@ test_data_t *init_test_data(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
|
|||||||
nstime_t time;
|
nstime_t time;
|
||||||
nstime_init(&time, 0);
|
nstime_init(&time, 0);
|
||||||
|
|
||||||
|
const size_t oversize_threshold = 8 * 1024 * 1024;
|
||||||
err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap,
|
err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap,
|
||||||
test_data->base, /* ind */ 1, &test_data->stats,
|
test_data->base, /* ind */ 1, &test_data->stats,
|
||||||
&test_data->stats_mtx, &time, dirty_decay_ms, muzzy_decay_ms);
|
&test_data->stats_mtx, &time, oversize_threshold, dirty_decay_ms,
|
||||||
|
muzzy_decay_ms);
|
||||||
assert_false(err, "");
|
assert_false(err, "");
|
||||||
|
|
||||||
return test_data;
|
return test_data;
|
||||||
|
Loading…
Reference in New Issue
Block a user