From fb327368db39a2edca5f9659a70a53bd3bb0ed6c Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Tue, 26 Jan 2021 18:35:18 -0800 Subject: [PATCH] SEC: Expand option configurability. This change pulls the SEC options into a struct, which simplifies their handling across various modules (e.g. PA needs to forward on SEC options from the malloc_conf string, but it doesn't really need to know their names). While we're here, make some of the fixed constants configurable, and unify naming from the configuration options to the internals. --- .../internal/jemalloc_internal_externs.h | 5 +- include/jemalloc/internal/pa.h | 2 +- include/jemalloc/internal/sec.h | 42 +------------ include/jemalloc/internal/sec_opts.h | 59 +++++++++++++++++ src/arena.c | 5 +- src/ctl.c | 27 ++++++-- src/jemalloc.c | 30 +++++---- src/pa.c | 5 +- src/sec.c | 63 ++++++++++--------- src/stats.c | 4 +- test/unit/mallctl.c | 4 +- test/unit/sec.c | 61 +++++++++++------- 12 files changed, 185 insertions(+), 122 deletions(-) create mode 100644 include/jemalloc/internal/sec_opts.h diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h index da693559..de5731fc 100644 --- a/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -3,6 +3,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/hpa_opts.h" +#include "jemalloc/internal/sec_opts.h" #include "jemalloc/internal/tsd_types.h" #include "jemalloc/internal/nstime.h" @@ -16,9 +17,7 @@ extern bool opt_trust_madvise; extern bool opt_confirm_conf; extern bool opt_hpa; extern hpa_shard_opts_t opt_hpa_opts; -extern size_t opt_hpa_sec_max_alloc; -extern size_t opt_hpa_sec_max_bytes; -extern size_t opt_hpa_sec_nshards; +extern sec_opts_t opt_hpa_sec_opts; extern const char *opt_junk; extern bool opt_junk_alloc; diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 6ded54f8..acb94eb6 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -131,7 +131,7 @@ bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, * that we can boot without worrying about the HPA, then turn it on in a0. */ bool pa_shard_enable_hpa(pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts, - size_t sec_nshards, size_t sec_alloc_max, size_t sec_bytes_max); + const sec_opts_t *hpa_sec_opts); /* * We stop using the HPA when custom extent hooks are installed, but still * redirect deallocations to it. diff --git a/include/jemalloc/internal/sec.h b/include/jemalloc/internal/sec.h index fadf4b61..ddcdfbdf 100644 --- a/include/jemalloc/internal/sec.h +++ b/include/jemalloc/internal/sec.h @@ -103,49 +103,11 @@ struct sec_s { pai_t pai; pai_t *fallback; - /* - * We'll automatically refuse to cache any objects in this sec if - * they're larger than alloc_max bytes. - */ - size_t alloc_max; - /* - * Exceeding this amount of cached extents in a shard causes *all* of - * the bins in that shard to be flushed. - */ - size_t bytes_max; - /* - * The number of bytes (in all bins) we flush down to when we exceed - * bytes_cur. We want this to be less than bytes_cur, because - * otherwise we could get into situations where a shard undergoing - * net-deallocation keeps bytes_cur very near to bytes_max, so that - * most deallocations get immediately forwarded to the underlying PAI - * implementation, defeating the point of the SEC. - * - * Currently this is just set to bytes_max / 2, but eventually can be - * configurable. - */ - size_t bytes_after_flush; - - /* - * When we can't satisfy an allocation out of the SEC because there are - * no available ones cached, we allocate multiple of that size out of - * the fallback allocator. Eventually we might want to do something - * cleverer, but for now we just grab a fixed number. - * - * For now, just the constant 4. Eventually, it should be configurable. - */ - size_t batch_fill_extra; - - /* - * We don't necessarily always use all the shards; requests are - * distributed across shards [0, nshards - 1). - */ - size_t nshards; + sec_opts_t opts; sec_shard_t shards[SEC_NSHARDS_MAX]; }; -bool sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t alloc_max, - size_t bytes_max); +bool sec_init(sec_t *sec, pai_t *fallback, const sec_opts_t *opts); void sec_flush(tsdn_t *tsdn, sec_t *sec); void sec_disable(tsdn_t *tsdn, sec_t *sec); diff --git a/include/jemalloc/internal/sec_opts.h b/include/jemalloc/internal/sec_opts.h new file mode 100644 index 00000000..91b6d0de --- /dev/null +++ b/include/jemalloc/internal/sec_opts.h @@ -0,0 +1,59 @@ +#ifndef JEMALLOC_INTERNAL_SEC_OPTS_H +#define JEMALLOC_INTERNAL_SEC_OPTS_H + +/* + * The configuration settings used by an sec_t. Morally, this is part of the + * SEC interface, but we put it here for header-ordering reasons. + */ + +typedef struct sec_opts_s sec_opts_t; +struct sec_opts_s { + /* + * We don't necessarily always use all the shards; requests are + * distributed across shards [0, nshards - 1). + */ + size_t nshards; + /* + * We'll automatically refuse to cache any objects in this sec if + * they're larger than max_alloc bytes, instead forwarding such objects + * directly to the fallback. + */ + size_t max_alloc; + /* + * Exceeding this amount of cached extents in a shard causes us to start + * flushing bins in that shard until we fall below bytes_after_flush. + */ + size_t max_bytes; + /* + * The number of bytes (in all bins) we flush down to when we exceed + * bytes_cur. We want this to be less than bytes_cur, because + * otherwise we could get into situations where a shard undergoing + * net-deallocation keeps bytes_cur very near to max_bytes, so that + * most deallocations get immediately forwarded to the underlying PAI + * implementation, defeating the point of the SEC. + */ + size_t bytes_after_flush; + /* + * When we can't satisfy an allocation out of the SEC because there are + * no available ones cached, we allocate multiple of that size out of + * the fallback allocator. Eventually we might want to do something + * cleverer, but for now we just grab a fixed number. + */ + size_t batch_fill_extra; +}; + +#define SEC_OPTS_DEFAULT { \ + /* nshards */ \ + 4, \ + /* max_alloc */ \ + 32 * 1024, \ + /* max_bytes */ \ + 256 * 1024, \ + /* bytes_after_flush */ \ + 128 * 1024, \ + /* batch_fill_extra */ \ + 0 \ +} + + +#endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */ diff --git a/src/arena.c b/src/arena.c index da0f1f02..f054f093 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1479,9 +1479,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { * so arena_hpa_global is not yet initialized. */ if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) { - if (pa_shard_enable_hpa(&arena->pa_shard, - &opt_hpa_opts, opt_hpa_sec_nshards, opt_hpa_sec_max_alloc, - opt_hpa_sec_max_bytes)) { + if (pa_shard_enable_hpa(&arena->pa_shard, &opt_hpa_opts, + &opt_hpa_sec_opts)) { goto label_error; } } diff --git a/src/ctl.c b/src/ctl.c index 4fc3ad07..663cf866 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -98,9 +98,11 @@ CTL_PROTO(opt_hpa_slab_max_alloc) CTL_PROTO(opt_hpa_hugification_threshold) CTL_PROTO(opt_hpa_dehugification_threshold) CTL_PROTO(opt_hpa_dirty_mult) +CTL_PROTO(opt_hpa_sec_nshards) CTL_PROTO(opt_hpa_sec_max_alloc) CTL_PROTO(opt_hpa_sec_max_bytes) -CTL_PROTO(opt_hpa_sec_nshards) +CTL_PROTO(opt_hpa_sec_bytes_after_flush) +CTL_PROTO(opt_hpa_sec_batch_fill_extra) CTL_PROTO(opt_metadata_thp) CTL_PROTO(opt_retain) CTL_PROTO(opt_dss) @@ -406,9 +408,13 @@ static const ctl_named_node_t opt_node[] = { {NAME("hpa_dehugification_threshold"), CTL(opt_hpa_dehugification_threshold)}, {NAME("hpa_dirty_mult"), CTL(opt_hpa_dirty_mult)}, + {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)}, {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)}, - {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, + {NAME("hpa_sec_bytes_after_flush"), + CTL(opt_hpa_sec_bytes_after_flush)}, + {NAME("hpa_sec_batch_fill_extra"), + CTL(opt_hpa_sec_batch_fill_extra)}, {NAME("metadata_thp"), CTL(opt_metadata_thp)}, {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)}, @@ -2100,8 +2106,9 @@ CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool) CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool) CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) + +/* HPA options. */ CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool) -CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t) CTL_RO_NL_GEN(opt_hpa_hugification_threshold, opt_hpa_opts.hugification_threshold, size_t) CTL_RO_NL_GEN(opt_hpa_dehugification_threshold, @@ -2111,9 +2118,17 @@ CTL_RO_NL_GEN(opt_hpa_dehugification_threshold, * its representation are internal implementation details. */ CTL_RO_NL_GEN(opt_hpa_dirty_mult, opt_hpa_opts.dirty_mult, fxp_t) -CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_max_alloc, size_t) -CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_max_bytes, size_t) -CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_nshards, size_t) +CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t) + +/* HPA SEC options */ +CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t) +CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t) +CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t) +CTL_RO_NL_GEN(opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush, + size_t) +CTL_RO_NL_GEN(opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, + size_t) + CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *) CTL_RO_NL_GEN(opt_retain, opt_retain, bool) diff --git a/src/jemalloc.c b/src/jemalloc.c index 125682bf..613733ff 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -153,11 +153,7 @@ malloc_mutex_t arenas_lock; /* The global hpa, and whether it's on. */ bool opt_hpa = false; hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT; - -size_t opt_hpa_sec_max_alloc = 32 * 1024; -/* These settings correspond to a maximum of 1MB cached per arena. */ -size_t opt_hpa_sec_max_bytes = 256 * 1024; -size_t opt_hpa_sec_nshards = 4; +sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT; /* * Arenas that are used to service external requests. Not all elements of the @@ -1473,12 +1469,21 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_CONTINUE; } - CONF_HANDLE_SIZE_T(opt_hpa_sec_max_alloc, "hpa_sec_max_alloc", - PAGE, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); - CONF_HANDLE_SIZE_T(opt_hpa_sec_max_bytes, "hpa_sec_max_bytes", - PAGE, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); - CONF_HANDLE_SIZE_T(opt_hpa_sec_nshards, "hpa_sec_nshards", - 0, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards, + "hpa_sec_nshards", 0, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_alloc, + "hpa_sec_max_alloc", PAGE, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes, + "hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush, + "hpa_sec_bytes_after_flush", PAGE, 0, + CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra, + "hpa_sec_batch_fill_extra", PAGE, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); if (CONF_MATCH("slab_sizes")) { if (CONF_MATCH_VALUE("default")) { @@ -1777,8 +1782,7 @@ malloc_init_hard_a0_locked() { } } else if (opt_hpa) { if (pa_shard_enable_hpa(&a0->pa_shard, &opt_hpa_opts, - opt_hpa_sec_nshards, opt_hpa_sec_max_alloc, - opt_hpa_sec_max_bytes)) { + &opt_hpa_sec_opts)) { return true; } } diff --git a/src/pa.c b/src/pa.c index abe3f00b..dd61aaa2 100644 --- a/src/pa.c +++ b/src/pa.c @@ -50,13 +50,12 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, bool pa_shard_enable_hpa(pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts, - size_t sec_nshards, size_t sec_alloc_max, size_t sec_bytes_max) { + const sec_opts_t *hpa_sec_opts) { if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base, &shard->edata_cache, shard->ind, hpa_opts)) { return true; } - if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, sec_nshards, - sec_alloc_max, sec_bytes_max)) { + if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, hpa_sec_opts)) { return true; } shard->ever_used_hpa = true; diff --git a/src/sec.c b/src/sec.c index f177bbee..c37cf35c 100644 --- a/src/sec.c +++ b/src/sec.c @@ -19,12 +19,12 @@ sec_bin_init(sec_bin_t *bin) { } bool -sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t alloc_max, - size_t bytes_max) { - if (nshards > SEC_NSHARDS_MAX) { - nshards = SEC_NSHARDS_MAX; +sec_init(sec_t *sec, pai_t *fallback, const sec_opts_t *opts) { + size_t nshards_clipped = opts->nshards; + if (nshards_clipped > SEC_NSHARDS_MAX) { + nshards_clipped = SEC_NSHARDS_MAX; } - for (size_t i = 0; i < nshards; i++) { + for (size_t i = 0; i < nshards_clipped; i++) { sec_shard_t *shard = &sec->shards[i]; bool err = malloc_mutex_init(&shard->mtx, "sec_shard", WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive); @@ -39,15 +39,15 @@ sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t alloc_max, shard->to_flush_next = 0; } sec->fallback = fallback; - sec->alloc_max = alloc_max; - if (sec->alloc_max > sz_pind2sz(SEC_NPSIZES - 1)) { - sec->alloc_max = sz_pind2sz(SEC_NPSIZES - 1); + + size_t max_alloc_clipped = opts->max_alloc; + if (max_alloc_clipped > sz_pind2sz(SEC_NPSIZES - 1)) { + max_alloc_clipped = sz_pind2sz(SEC_NPSIZES - 1); } - sec->bytes_max = bytes_max; - sec->bytes_after_flush = bytes_max / 2; - sec->batch_fill_extra = 4; - sec->nshards = nshards; + sec->opts = *opts; + sec->opts.nshards = nshards_clipped; + sec->opts.max_alloc = max_alloc_clipped; /* * Initialize these last so that an improper use of an SEC whose @@ -83,8 +83,9 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { * when we multiply by the number of shards. */ uint64_t rand32 = prng_lg_range_u64(tsd_prng_statep_get(tsd), 32); - uint32_t idx = (uint32_t)((rand32 * (uint64_t)sec->nshards) >> 32); - assert(idx < (uint32_t)sec->nshards); + uint32_t idx = + (uint32_t)((rand32 * (uint64_t)sec->opts.nshards) >> 32); + assert(idx < (uint32_t)sec->opts.nshards); *idxp = (uint8_t)idx; } return &sec->shards[*idxp]; @@ -99,7 +100,7 @@ sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) { malloc_mutex_assert_owner(tsdn, &shard->mtx); edata_list_active_t to_flush; edata_list_active_init(&to_flush); - while (shard->bytes_cur > sec->bytes_after_flush) { + while (shard->bytes_cur > sec->opts.bytes_after_flush) { /* Pick a victim. */ sec_bin_t *bin = &shard->bins[shard->to_flush_next]; @@ -155,7 +156,7 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_list_active_t result; edata_list_active_init(&result); size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size, - 1 + sec->batch_fill_extra, &result); + 1 + sec->opts.batch_fill_extra, &result); edata_t *ret = edata_list_active_first(&result); if (ret != NULL) { @@ -182,7 +183,7 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, bin->bytes_cur += new_cached_bytes; shard->bytes_cur += new_cached_bytes; - if (shard->bytes_cur > sec->bytes_max) { + if (shard->bytes_cur > sec->opts.max_bytes) { sec_flush_some_and_unlock(tsdn, sec, shard); } else { malloc_mutex_unlock(tsdn, &shard->mtx); @@ -197,8 +198,8 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) { sec_t *sec = (sec_t *)self; - if (zero || alignment > PAGE || sec->nshards == 0 - || size > sec->alloc_max) { + if (zero || alignment > PAGE || sec->opts.nshards == 0 + || size > sec->opts.max_alloc) { return pai_alloc(tsdn, sec->fallback, size, alignment, zero); } pszind_t pszind = sz_psz2ind(size); @@ -209,7 +210,8 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) { malloc_mutex_lock(tsdn, &shard->mtx); edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin); if (edata == NULL) { - if (!bin->being_batch_filled && sec->batch_fill_extra > 0) { + if (!bin->being_batch_filled + && sec->opts.batch_fill_extra > 0) { bin->being_batch_filled = true; do_batch_fill = true; } @@ -266,7 +268,7 @@ static void sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_t *edata) { malloc_mutex_assert_owner(tsdn, &shard->mtx); - assert(shard->bytes_cur <= sec->bytes_max); + assert(shard->bytes_cur <= sec->opts.max_bytes); size_t size = edata_size_get(edata); pszind_t pszind = sz_psz2ind(size); /* @@ -277,7 +279,7 @@ sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_list_active_prepend(&bin->freelist, edata); bin->bytes_cur += size; shard->bytes_cur += size; - if (shard->bytes_cur > sec->bytes_max) { + if (shard->bytes_cur > sec->opts.max_bytes) { /* * We've exceeded the shard limit. We make two nods in the * direction of fragmentation avoidance: we flush everything in @@ -297,7 +299,8 @@ sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, static void sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { sec_t *sec = (sec_t *)self; - if (sec->nshards == 0 || edata_size_get(edata) > sec->alloc_max) { + if (sec->opts.nshards == 0 + || edata_size_get(edata) > sec->opts.max_alloc) { pai_dalloc(tsdn, sec->fallback, edata); return; } @@ -313,7 +316,7 @@ sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { void sec_flush(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_lock(tsdn, &sec->shards[i].mtx); sec_flush_all_locked(tsdn, sec, &sec->shards[i]); malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); @@ -322,7 +325,7 @@ sec_flush(tsdn_t *tsdn, sec_t *sec) { void sec_disable(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_lock(tsdn, &sec->shards[i].mtx); sec->shards[i].enabled = false; sec_flush_all_locked(tsdn, sec, &sec->shards[i]); @@ -333,7 +336,7 @@ sec_disable(tsdn_t *tsdn, sec_t *sec) { void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) { size_t sum = 0; - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { /* * We could save these lock acquisitions by making bytes_cur * atomic, but stats collection is rare anyways and we expect @@ -349,7 +352,7 @@ sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) { void sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_lock(tsdn, &sec->shards[i].mtx); malloc_mutex_prof_accum(tsdn, mutex_prof_data, &sec->shards[i].mtx); @@ -359,21 +362,21 @@ sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, void sec_prefork2(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_prefork(tsdn, &sec->shards[i].mtx); } } void sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx); } } void sec_postfork_child(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx); } } diff --git a/src/stats.c b/src/stats.c index 7a0526c5..69cb2d3e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1486,9 +1486,11 @@ stats_general_print(emitter_t *emitter) { "opt.hpa_dirty_mult", emitter_type_string, &bufp); } } + OPT_WRITE_SIZE_T("hpa_sec_nshards") OPT_WRITE_SIZE_T("hpa_sec_max_alloc") OPT_WRITE_SIZE_T("hpa_sec_max_bytes") - OPT_WRITE_SIZE_T("hpa_sec_nshards") + OPT_WRITE_SIZE_T("hpa_sec_bytes_after_flush") + OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra") OPT_WRITE_CHAR_P("metadata_thp") OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread") OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms") diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 1fb74667..e9e0feb6 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -287,9 +287,11 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(const char *, dss, always); TEST_MALLCTL_OPT(bool, hpa, always); TEST_MALLCTL_OPT(size_t, hpa_slab_max_alloc, always); + TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always); TEST_MALLCTL_OPT(size_t, hpa_sec_max_alloc, always); TEST_MALLCTL_OPT(size_t, hpa_sec_max_bytes, always); - TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always); + TEST_MALLCTL_OPT(size_t, hpa_sec_bytes_after_flush, always); + TEST_MALLCTL_OPT(size_t, hpa_sec_batch_fill_extra, always); TEST_MALLCTL_OPT(unsigned, narenas, always); TEST_MALLCTL_OPT(const char *, percpu_arena, always); TEST_MALLCTL_OPT(size_t, oversize_threshold, always); diff --git a/test/unit/sec.c b/test/unit/sec.c index ff39453c..36ae1a52 100644 --- a/test/unit/sec.c +++ b/test/unit/sec.c @@ -23,6 +23,24 @@ struct pai_test_allocator_s { bool shrink_return_value; }; +static void +test_sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t max_alloc, + size_t max_bytes) { + sec_opts_t opts; + opts.nshards = 1; + opts.max_alloc = max_alloc; + opts.max_bytes = max_bytes; + /* + * Just choose reasonable defaults for these; most tests don't care so + * long as they're something reasonable. + */ + opts.bytes_after_flush = max_bytes / 2; + opts.batch_fill_extra = 4; + + bool err = sec_init(sec, fallback, &opts); + assert_false(err, "Unexpected initialization failure"); +} + static inline edata_t * pai_test_allocator_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) { @@ -143,8 +161,8 @@ TEST_BEGIN(test_reuse) { enum { NALLOCS = 11 }; edata_t *one_page[NALLOCS]; edata_t *two_page[NALLOCS]; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ 2 * PAGE, - /* bytes_max */ 2 * (NALLOCS * PAGE + NALLOCS * 2 * PAGE)); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ 2 * PAGE, + /* max_bytes */ 2 * (NALLOCS * PAGE + NALLOCS * 2 * PAGE)); for (int i = 0; i < NALLOCS; i++) { one_page[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); @@ -213,8 +231,8 @@ TEST_BEGIN(test_auto_flush) { enum { NALLOCS = 10 }; edata_t *extra_alloc; edata_t *allocs[NALLOCS]; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ NALLOCS * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ NALLOCS * PAGE); for (int i = 0; i < NALLOCS; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); @@ -266,8 +284,8 @@ do_disable_flush_test(bool is_disable) { enum { NALLOCS = 11 }; edata_t *allocs[NALLOCS]; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ NALLOCS * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ NALLOCS * PAGE); for (int i = 0; i < NALLOCS; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); @@ -321,18 +339,18 @@ TEST_BEGIN(test_flush) { } TEST_END -TEST_BEGIN(test_alloc_max_respected) { +TEST_BEGIN(test_max_alloc_respected) { pai_test_allocator_t ta; pai_test_allocator_init(&ta); sec_t sec; /* See the note above -- we can't use the real tsd. */ tsdn_t *tsdn = TSDN_NULL; - size_t alloc_max = 2 * PAGE; + size_t max_alloc = 2 * PAGE; size_t attempted_alloc = 3 * PAGE; - sec_init(&sec, &ta.pai, /* nshards */ 1, alloc_max, - /* bytes_max */ 1000 * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, max_alloc, + /* max_bytes */ 1000 * PAGE); for (size_t i = 0; i < 100; i++) { expect_zu_eq(i, ta.alloc_count, @@ -362,8 +380,8 @@ TEST_BEGIN(test_expand_shrink_delegate) { /* See the note above -- we can't use the real tsd. */ tsdn_t *tsdn = TSDN_NULL; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ 10 * PAGE, - /* bytes_max */ 1000 * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ 10 * PAGE, + /* max_bytes */ 1000 * PAGE); edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); expect_ptr_not_null(edata, "Unexpected alloc failure"); @@ -395,8 +413,9 @@ TEST_BEGIN(test_nshards_0) { /* See the note above -- we can't use the real tsd. */ tsdn_t *tsdn = TSDN_NULL; - sec_init(&sec, &ta.pai, /* nshards */ 0, /* alloc_max */ 10 * PAGE, - /* bytes_max */ 1000 * PAGE); + sec_opts_t opts = SEC_OPTS_DEFAULT; + opts.nshards = 0; + sec_init(&sec, &ta.pai, &opts); edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); @@ -433,8 +452,8 @@ TEST_BEGIN(test_stats_simple) { FLUSH_PAGES = 20, }; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ FLUSH_PAGES * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ FLUSH_PAGES * PAGE); edata_t *allocs[FLUSH_PAGES]; for (size_t i = 0; i < FLUSH_PAGES; i++) { @@ -470,8 +489,8 @@ TEST_BEGIN(test_stats_auto_flush) { FLUSH_PAGES = 10, }; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ FLUSH_PAGES * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ FLUSH_PAGES * PAGE); edata_t *extra_alloc0; edata_t *extra_alloc1; @@ -514,8 +533,8 @@ TEST_BEGIN(test_stats_manual_flush) { FLUSH_PAGES = 10, }; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ FLUSH_PAGES * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ FLUSH_PAGES * PAGE); edata_t *allocs[FLUSH_PAGES]; for (size_t i = 0; i < FLUSH_PAGES; i++) { @@ -550,7 +569,7 @@ main(void) { test_auto_flush, test_disable, test_flush, - test_alloc_max_respected, + test_max_alloc_respected, test_expand_shrink_delegate, test_nshards_0, test_stats_simple,