diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h index da693559..de5731fc 100644 --- a/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -3,6 +3,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/hpa_opts.h" +#include "jemalloc/internal/sec_opts.h" #include "jemalloc/internal/tsd_types.h" #include "jemalloc/internal/nstime.h" @@ -16,9 +17,7 @@ extern bool opt_trust_madvise; extern bool opt_confirm_conf; extern bool opt_hpa; extern hpa_shard_opts_t opt_hpa_opts; -extern size_t opt_hpa_sec_max_alloc; -extern size_t opt_hpa_sec_max_bytes; -extern size_t opt_hpa_sec_nshards; +extern sec_opts_t opt_hpa_sec_opts; extern const char *opt_junk; extern bool opt_junk_alloc; diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 6ded54f8..acb94eb6 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -131,7 +131,7 @@ bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, * that we can boot without worrying about the HPA, then turn it on in a0. */ bool pa_shard_enable_hpa(pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts, - size_t sec_nshards, size_t sec_alloc_max, size_t sec_bytes_max); + const sec_opts_t *hpa_sec_opts); /* * We stop using the HPA when custom extent hooks are installed, but still * redirect deallocations to it. diff --git a/include/jemalloc/internal/sec.h b/include/jemalloc/internal/sec.h index fadf4b61..ddcdfbdf 100644 --- a/include/jemalloc/internal/sec.h +++ b/include/jemalloc/internal/sec.h @@ -103,49 +103,11 @@ struct sec_s { pai_t pai; pai_t *fallback; - /* - * We'll automatically refuse to cache any objects in this sec if - * they're larger than alloc_max bytes. - */ - size_t alloc_max; - /* - * Exceeding this amount of cached extents in a shard causes *all* of - * the bins in that shard to be flushed. - */ - size_t bytes_max; - /* - * The number of bytes (in all bins) we flush down to when we exceed - * bytes_cur. We want this to be less than bytes_cur, because - * otherwise we could get into situations where a shard undergoing - * net-deallocation keeps bytes_cur very near to bytes_max, so that - * most deallocations get immediately forwarded to the underlying PAI - * implementation, defeating the point of the SEC. - * - * Currently this is just set to bytes_max / 2, but eventually can be - * configurable. - */ - size_t bytes_after_flush; - - /* - * When we can't satisfy an allocation out of the SEC because there are - * no available ones cached, we allocate multiple of that size out of - * the fallback allocator. Eventually we might want to do something - * cleverer, but for now we just grab a fixed number. - * - * For now, just the constant 4. Eventually, it should be configurable. - */ - size_t batch_fill_extra; - - /* - * We don't necessarily always use all the shards; requests are - * distributed across shards [0, nshards - 1). - */ - size_t nshards; + sec_opts_t opts; sec_shard_t shards[SEC_NSHARDS_MAX]; }; -bool sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t alloc_max, - size_t bytes_max); +bool sec_init(sec_t *sec, pai_t *fallback, const sec_opts_t *opts); void sec_flush(tsdn_t *tsdn, sec_t *sec); void sec_disable(tsdn_t *tsdn, sec_t *sec); diff --git a/include/jemalloc/internal/sec_opts.h b/include/jemalloc/internal/sec_opts.h new file mode 100644 index 00000000..91b6d0de --- /dev/null +++ b/include/jemalloc/internal/sec_opts.h @@ -0,0 +1,59 @@ +#ifndef JEMALLOC_INTERNAL_SEC_OPTS_H +#define JEMALLOC_INTERNAL_SEC_OPTS_H + +/* + * The configuration settings used by an sec_t. Morally, this is part of the + * SEC interface, but we put it here for header-ordering reasons. + */ + +typedef struct sec_opts_s sec_opts_t; +struct sec_opts_s { + /* + * We don't necessarily always use all the shards; requests are + * distributed across shards [0, nshards - 1). + */ + size_t nshards; + /* + * We'll automatically refuse to cache any objects in this sec if + * they're larger than max_alloc bytes, instead forwarding such objects + * directly to the fallback. + */ + size_t max_alloc; + /* + * Exceeding this amount of cached extents in a shard causes us to start + * flushing bins in that shard until we fall below bytes_after_flush. + */ + size_t max_bytes; + /* + * The number of bytes (in all bins) we flush down to when we exceed + * bytes_cur. We want this to be less than bytes_cur, because + * otherwise we could get into situations where a shard undergoing + * net-deallocation keeps bytes_cur very near to max_bytes, so that + * most deallocations get immediately forwarded to the underlying PAI + * implementation, defeating the point of the SEC. + */ + size_t bytes_after_flush; + /* + * When we can't satisfy an allocation out of the SEC because there are + * no available ones cached, we allocate multiple of that size out of + * the fallback allocator. Eventually we might want to do something + * cleverer, but for now we just grab a fixed number. + */ + size_t batch_fill_extra; +}; + +#define SEC_OPTS_DEFAULT { \ + /* nshards */ \ + 4, \ + /* max_alloc */ \ + 32 * 1024, \ + /* max_bytes */ \ + 256 * 1024, \ + /* bytes_after_flush */ \ + 128 * 1024, \ + /* batch_fill_extra */ \ + 0 \ +} + + +#endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */ diff --git a/src/arena.c b/src/arena.c index da0f1f02..f054f093 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1479,9 +1479,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { * so arena_hpa_global is not yet initialized. */ if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) { - if (pa_shard_enable_hpa(&arena->pa_shard, - &opt_hpa_opts, opt_hpa_sec_nshards, opt_hpa_sec_max_alloc, - opt_hpa_sec_max_bytes)) { + if (pa_shard_enable_hpa(&arena->pa_shard, &opt_hpa_opts, + &opt_hpa_sec_opts)) { goto label_error; } } diff --git a/src/ctl.c b/src/ctl.c index 4fc3ad07..663cf866 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -98,9 +98,11 @@ CTL_PROTO(opt_hpa_slab_max_alloc) CTL_PROTO(opt_hpa_hugification_threshold) CTL_PROTO(opt_hpa_dehugification_threshold) CTL_PROTO(opt_hpa_dirty_mult) +CTL_PROTO(opt_hpa_sec_nshards) CTL_PROTO(opt_hpa_sec_max_alloc) CTL_PROTO(opt_hpa_sec_max_bytes) -CTL_PROTO(opt_hpa_sec_nshards) +CTL_PROTO(opt_hpa_sec_bytes_after_flush) +CTL_PROTO(opt_hpa_sec_batch_fill_extra) CTL_PROTO(opt_metadata_thp) CTL_PROTO(opt_retain) CTL_PROTO(opt_dss) @@ -406,9 +408,13 @@ static const ctl_named_node_t opt_node[] = { {NAME("hpa_dehugification_threshold"), CTL(opt_hpa_dehugification_threshold)}, {NAME("hpa_dirty_mult"), CTL(opt_hpa_dirty_mult)}, + {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)}, {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)}, - {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, + {NAME("hpa_sec_bytes_after_flush"), + CTL(opt_hpa_sec_bytes_after_flush)}, + {NAME("hpa_sec_batch_fill_extra"), + CTL(opt_hpa_sec_batch_fill_extra)}, {NAME("metadata_thp"), CTL(opt_metadata_thp)}, {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)}, @@ -2100,8 +2106,9 @@ CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool) CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool) CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) + +/* HPA options. */ CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool) -CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t) CTL_RO_NL_GEN(opt_hpa_hugification_threshold, opt_hpa_opts.hugification_threshold, size_t) CTL_RO_NL_GEN(opt_hpa_dehugification_threshold, @@ -2111,9 +2118,17 @@ CTL_RO_NL_GEN(opt_hpa_dehugification_threshold, * its representation are internal implementation details. */ CTL_RO_NL_GEN(opt_hpa_dirty_mult, opt_hpa_opts.dirty_mult, fxp_t) -CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_max_alloc, size_t) -CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_max_bytes, size_t) -CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_nshards, size_t) +CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t) + +/* HPA SEC options */ +CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t) +CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t) +CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t) +CTL_RO_NL_GEN(opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush, + size_t) +CTL_RO_NL_GEN(opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, + size_t) + CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *) CTL_RO_NL_GEN(opt_retain, opt_retain, bool) diff --git a/src/jemalloc.c b/src/jemalloc.c index 125682bf..613733ff 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -153,11 +153,7 @@ malloc_mutex_t arenas_lock; /* The global hpa, and whether it's on. */ bool opt_hpa = false; hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT; - -size_t opt_hpa_sec_max_alloc = 32 * 1024; -/* These settings correspond to a maximum of 1MB cached per arena. */ -size_t opt_hpa_sec_max_bytes = 256 * 1024; -size_t opt_hpa_sec_nshards = 4; +sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT; /* * Arenas that are used to service external requests. Not all elements of the @@ -1473,12 +1469,21 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_CONTINUE; } - CONF_HANDLE_SIZE_T(opt_hpa_sec_max_alloc, "hpa_sec_max_alloc", - PAGE, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); - CONF_HANDLE_SIZE_T(opt_hpa_sec_max_bytes, "hpa_sec_max_bytes", - PAGE, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); - CONF_HANDLE_SIZE_T(opt_hpa_sec_nshards, "hpa_sec_nshards", - 0, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards, + "hpa_sec_nshards", 0, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_alloc, + "hpa_sec_max_alloc", PAGE, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes, + "hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush, + "hpa_sec_bytes_after_flush", PAGE, 0, + CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra, + "hpa_sec_batch_fill_extra", PAGE, 0, CONF_CHECK_MIN, + CONF_DONT_CHECK_MAX, true); if (CONF_MATCH("slab_sizes")) { if (CONF_MATCH_VALUE("default")) { @@ -1777,8 +1782,7 @@ malloc_init_hard_a0_locked() { } } else if (opt_hpa) { if (pa_shard_enable_hpa(&a0->pa_shard, &opt_hpa_opts, - opt_hpa_sec_nshards, opt_hpa_sec_max_alloc, - opt_hpa_sec_max_bytes)) { + &opt_hpa_sec_opts)) { return true; } } diff --git a/src/pa.c b/src/pa.c index abe3f00b..dd61aaa2 100644 --- a/src/pa.c +++ b/src/pa.c @@ -50,13 +50,12 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, bool pa_shard_enable_hpa(pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts, - size_t sec_nshards, size_t sec_alloc_max, size_t sec_bytes_max) { + const sec_opts_t *hpa_sec_opts) { if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base, &shard->edata_cache, shard->ind, hpa_opts)) { return true; } - if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, sec_nshards, - sec_alloc_max, sec_bytes_max)) { + if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, hpa_sec_opts)) { return true; } shard->ever_used_hpa = true; diff --git a/src/sec.c b/src/sec.c index f177bbee..c37cf35c 100644 --- a/src/sec.c +++ b/src/sec.c @@ -19,12 +19,12 @@ sec_bin_init(sec_bin_t *bin) { } bool -sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t alloc_max, - size_t bytes_max) { - if (nshards > SEC_NSHARDS_MAX) { - nshards = SEC_NSHARDS_MAX; +sec_init(sec_t *sec, pai_t *fallback, const sec_opts_t *opts) { + size_t nshards_clipped = opts->nshards; + if (nshards_clipped > SEC_NSHARDS_MAX) { + nshards_clipped = SEC_NSHARDS_MAX; } - for (size_t i = 0; i < nshards; i++) { + for (size_t i = 0; i < nshards_clipped; i++) { sec_shard_t *shard = &sec->shards[i]; bool err = malloc_mutex_init(&shard->mtx, "sec_shard", WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive); @@ -39,15 +39,15 @@ sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t alloc_max, shard->to_flush_next = 0; } sec->fallback = fallback; - sec->alloc_max = alloc_max; - if (sec->alloc_max > sz_pind2sz(SEC_NPSIZES - 1)) { - sec->alloc_max = sz_pind2sz(SEC_NPSIZES - 1); + + size_t max_alloc_clipped = opts->max_alloc; + if (max_alloc_clipped > sz_pind2sz(SEC_NPSIZES - 1)) { + max_alloc_clipped = sz_pind2sz(SEC_NPSIZES - 1); } - sec->bytes_max = bytes_max; - sec->bytes_after_flush = bytes_max / 2; - sec->batch_fill_extra = 4; - sec->nshards = nshards; + sec->opts = *opts; + sec->opts.nshards = nshards_clipped; + sec->opts.max_alloc = max_alloc_clipped; /* * Initialize these last so that an improper use of an SEC whose @@ -83,8 +83,9 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { * when we multiply by the number of shards. */ uint64_t rand32 = prng_lg_range_u64(tsd_prng_statep_get(tsd), 32); - uint32_t idx = (uint32_t)((rand32 * (uint64_t)sec->nshards) >> 32); - assert(idx < (uint32_t)sec->nshards); + uint32_t idx = + (uint32_t)((rand32 * (uint64_t)sec->opts.nshards) >> 32); + assert(idx < (uint32_t)sec->opts.nshards); *idxp = (uint8_t)idx; } return &sec->shards[*idxp]; @@ -99,7 +100,7 @@ sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) { malloc_mutex_assert_owner(tsdn, &shard->mtx); edata_list_active_t to_flush; edata_list_active_init(&to_flush); - while (shard->bytes_cur > sec->bytes_after_flush) { + while (shard->bytes_cur > sec->opts.bytes_after_flush) { /* Pick a victim. */ sec_bin_t *bin = &shard->bins[shard->to_flush_next]; @@ -155,7 +156,7 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_list_active_t result; edata_list_active_init(&result); size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size, - 1 + sec->batch_fill_extra, &result); + 1 + sec->opts.batch_fill_extra, &result); edata_t *ret = edata_list_active_first(&result); if (ret != NULL) { @@ -182,7 +183,7 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, bin->bytes_cur += new_cached_bytes; shard->bytes_cur += new_cached_bytes; - if (shard->bytes_cur > sec->bytes_max) { + if (shard->bytes_cur > sec->opts.max_bytes) { sec_flush_some_and_unlock(tsdn, sec, shard); } else { malloc_mutex_unlock(tsdn, &shard->mtx); @@ -197,8 +198,8 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) { sec_t *sec = (sec_t *)self; - if (zero || alignment > PAGE || sec->nshards == 0 - || size > sec->alloc_max) { + if (zero || alignment > PAGE || sec->opts.nshards == 0 + || size > sec->opts.max_alloc) { return pai_alloc(tsdn, sec->fallback, size, alignment, zero); } pszind_t pszind = sz_psz2ind(size); @@ -209,7 +210,8 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) { malloc_mutex_lock(tsdn, &shard->mtx); edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin); if (edata == NULL) { - if (!bin->being_batch_filled && sec->batch_fill_extra > 0) { + if (!bin->being_batch_filled + && sec->opts.batch_fill_extra > 0) { bin->being_batch_filled = true; do_batch_fill = true; } @@ -266,7 +268,7 @@ static void sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_t *edata) { malloc_mutex_assert_owner(tsdn, &shard->mtx); - assert(shard->bytes_cur <= sec->bytes_max); + assert(shard->bytes_cur <= sec->opts.max_bytes); size_t size = edata_size_get(edata); pszind_t pszind = sz_psz2ind(size); /* @@ -277,7 +279,7 @@ sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_list_active_prepend(&bin->freelist, edata); bin->bytes_cur += size; shard->bytes_cur += size; - if (shard->bytes_cur > sec->bytes_max) { + if (shard->bytes_cur > sec->opts.max_bytes) { /* * We've exceeded the shard limit. We make two nods in the * direction of fragmentation avoidance: we flush everything in @@ -297,7 +299,8 @@ sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, static void sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { sec_t *sec = (sec_t *)self; - if (sec->nshards == 0 || edata_size_get(edata) > sec->alloc_max) { + if (sec->opts.nshards == 0 + || edata_size_get(edata) > sec->opts.max_alloc) { pai_dalloc(tsdn, sec->fallback, edata); return; } @@ -313,7 +316,7 @@ sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { void sec_flush(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_lock(tsdn, &sec->shards[i].mtx); sec_flush_all_locked(tsdn, sec, &sec->shards[i]); malloc_mutex_unlock(tsdn, &sec->shards[i].mtx); @@ -322,7 +325,7 @@ sec_flush(tsdn_t *tsdn, sec_t *sec) { void sec_disable(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_lock(tsdn, &sec->shards[i].mtx); sec->shards[i].enabled = false; sec_flush_all_locked(tsdn, sec, &sec->shards[i]); @@ -333,7 +336,7 @@ sec_disable(tsdn_t *tsdn, sec_t *sec) { void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) { size_t sum = 0; - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { /* * We could save these lock acquisitions by making bytes_cur * atomic, but stats collection is rare anyways and we expect @@ -349,7 +352,7 @@ sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) { void sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_lock(tsdn, &sec->shards[i].mtx); malloc_mutex_prof_accum(tsdn, mutex_prof_data, &sec->shards[i].mtx); @@ -359,21 +362,21 @@ sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, void sec_prefork2(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_prefork(tsdn, &sec->shards[i].mtx); } } void sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx); } } void sec_postfork_child(tsdn_t *tsdn, sec_t *sec) { - for (size_t i = 0; i < sec->nshards; i++) { + for (size_t i = 0; i < sec->opts.nshards; i++) { malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx); } } diff --git a/src/stats.c b/src/stats.c index 7a0526c5..69cb2d3e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1486,9 +1486,11 @@ stats_general_print(emitter_t *emitter) { "opt.hpa_dirty_mult", emitter_type_string, &bufp); } } + OPT_WRITE_SIZE_T("hpa_sec_nshards") OPT_WRITE_SIZE_T("hpa_sec_max_alloc") OPT_WRITE_SIZE_T("hpa_sec_max_bytes") - OPT_WRITE_SIZE_T("hpa_sec_nshards") + OPT_WRITE_SIZE_T("hpa_sec_bytes_after_flush") + OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra") OPT_WRITE_CHAR_P("metadata_thp") OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread") OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms") diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 1fb74667..e9e0feb6 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -287,9 +287,11 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(const char *, dss, always); TEST_MALLCTL_OPT(bool, hpa, always); TEST_MALLCTL_OPT(size_t, hpa_slab_max_alloc, always); + TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always); TEST_MALLCTL_OPT(size_t, hpa_sec_max_alloc, always); TEST_MALLCTL_OPT(size_t, hpa_sec_max_bytes, always); - TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always); + TEST_MALLCTL_OPT(size_t, hpa_sec_bytes_after_flush, always); + TEST_MALLCTL_OPT(size_t, hpa_sec_batch_fill_extra, always); TEST_MALLCTL_OPT(unsigned, narenas, always); TEST_MALLCTL_OPT(const char *, percpu_arena, always); TEST_MALLCTL_OPT(size_t, oversize_threshold, always); diff --git a/test/unit/sec.c b/test/unit/sec.c index ff39453c..36ae1a52 100644 --- a/test/unit/sec.c +++ b/test/unit/sec.c @@ -23,6 +23,24 @@ struct pai_test_allocator_s { bool shrink_return_value; }; +static void +test_sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t max_alloc, + size_t max_bytes) { + sec_opts_t opts; + opts.nshards = 1; + opts.max_alloc = max_alloc; + opts.max_bytes = max_bytes; + /* + * Just choose reasonable defaults for these; most tests don't care so + * long as they're something reasonable. + */ + opts.bytes_after_flush = max_bytes / 2; + opts.batch_fill_extra = 4; + + bool err = sec_init(sec, fallback, &opts); + assert_false(err, "Unexpected initialization failure"); +} + static inline edata_t * pai_test_allocator_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) { @@ -143,8 +161,8 @@ TEST_BEGIN(test_reuse) { enum { NALLOCS = 11 }; edata_t *one_page[NALLOCS]; edata_t *two_page[NALLOCS]; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ 2 * PAGE, - /* bytes_max */ 2 * (NALLOCS * PAGE + NALLOCS * 2 * PAGE)); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ 2 * PAGE, + /* max_bytes */ 2 * (NALLOCS * PAGE + NALLOCS * 2 * PAGE)); for (int i = 0; i < NALLOCS; i++) { one_page[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); @@ -213,8 +231,8 @@ TEST_BEGIN(test_auto_flush) { enum { NALLOCS = 10 }; edata_t *extra_alloc; edata_t *allocs[NALLOCS]; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ NALLOCS * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ NALLOCS * PAGE); for (int i = 0; i < NALLOCS; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); @@ -266,8 +284,8 @@ do_disable_flush_test(bool is_disable) { enum { NALLOCS = 11 }; edata_t *allocs[NALLOCS]; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ NALLOCS * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ NALLOCS * PAGE); for (int i = 0; i < NALLOCS; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); @@ -321,18 +339,18 @@ TEST_BEGIN(test_flush) { } TEST_END -TEST_BEGIN(test_alloc_max_respected) { +TEST_BEGIN(test_max_alloc_respected) { pai_test_allocator_t ta; pai_test_allocator_init(&ta); sec_t sec; /* See the note above -- we can't use the real tsd. */ tsdn_t *tsdn = TSDN_NULL; - size_t alloc_max = 2 * PAGE; + size_t max_alloc = 2 * PAGE; size_t attempted_alloc = 3 * PAGE; - sec_init(&sec, &ta.pai, /* nshards */ 1, alloc_max, - /* bytes_max */ 1000 * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, max_alloc, + /* max_bytes */ 1000 * PAGE); for (size_t i = 0; i < 100; i++) { expect_zu_eq(i, ta.alloc_count, @@ -362,8 +380,8 @@ TEST_BEGIN(test_expand_shrink_delegate) { /* See the note above -- we can't use the real tsd. */ tsdn_t *tsdn = TSDN_NULL; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ 10 * PAGE, - /* bytes_max */ 1000 * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ 10 * PAGE, + /* max_bytes */ 1000 * PAGE); edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); expect_ptr_not_null(edata, "Unexpected alloc failure"); @@ -395,8 +413,9 @@ TEST_BEGIN(test_nshards_0) { /* See the note above -- we can't use the real tsd. */ tsdn_t *tsdn = TSDN_NULL; - sec_init(&sec, &ta.pai, /* nshards */ 0, /* alloc_max */ 10 * PAGE, - /* bytes_max */ 1000 * PAGE); + sec_opts_t opts = SEC_OPTS_DEFAULT; + opts.nshards = 0; + sec_init(&sec, &ta.pai, &opts); edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false); @@ -433,8 +452,8 @@ TEST_BEGIN(test_stats_simple) { FLUSH_PAGES = 20, }; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ FLUSH_PAGES * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ FLUSH_PAGES * PAGE); edata_t *allocs[FLUSH_PAGES]; for (size_t i = 0; i < FLUSH_PAGES; i++) { @@ -470,8 +489,8 @@ TEST_BEGIN(test_stats_auto_flush) { FLUSH_PAGES = 10, }; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ FLUSH_PAGES * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ FLUSH_PAGES * PAGE); edata_t *extra_alloc0; edata_t *extra_alloc1; @@ -514,8 +533,8 @@ TEST_BEGIN(test_stats_manual_flush) { FLUSH_PAGES = 10, }; - sec_init(&sec, &ta.pai, /* nshards */ 1, /* alloc_max */ PAGE, - /* bytes_max */ FLUSH_PAGES * PAGE); + test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ PAGE, + /* max_bytes */ FLUSH_PAGES * PAGE); edata_t *allocs[FLUSH_PAGES]; for (size_t i = 0; i < FLUSH_PAGES; i++) { @@ -550,7 +569,7 @@ main(void) { test_auto_flush, test_disable, test_flush, - test_alloc_max_respected, + test_max_alloc_respected, test_expand_shrink_delegate, test_nshards_0, test_stats_simple,