diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h index c26153e3..b152068a 100644 --- a/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -13,6 +13,8 @@ extern bool opt_abort; extern bool opt_abort_conf; extern bool opt_confirm_conf; extern bool opt_hpa; +extern size_t opt_hpa_slab_goal; +extern size_t opt_hpa_slab_max_alloc; extern const char *opt_junk; extern bool opt_junk_alloc; extern bool opt_junk_free; diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 7f73c274..eced8cae 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -123,7 +123,8 @@ bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, * This isn't exposed to users; we allow late enablement of the HPA shard so * that we can boot without worrying about the HPA, then turn it on in a0. */ -bool pa_shard_enable_hpa(pa_shard_t *shard, hpa_t *hpa); +bool pa_shard_enable_hpa(pa_shard_t *shard, hpa_t *hpa, size_t ps_goal, + size_t ps_alloc_max); /* * We stop using the HPA when custom extent hooks are installed, but still * redirect deallocations to it. diff --git a/src/arena.c b/src/arena.c index 74f90ccc..3403526e 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1527,7 +1527,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { * so arena_hpa_global is not yet initialized. */ if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) { - if (pa_shard_enable_hpa(&arena->pa_shard, &arena_hpa_global)) { + if (pa_shard_enable_hpa(&arena->pa_shard, &arena_hpa_global, + opt_hpa_slab_goal, opt_hpa_slab_max_alloc)) { goto label_error; } } diff --git a/src/ctl.c b/src/ctl.c index 9e22e66c..fe6e8440 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -91,6 +91,8 @@ CTL_PROTO(opt_abort) CTL_PROTO(opt_abort_conf) CTL_PROTO(opt_confirm_conf) CTL_PROTO(opt_hpa) +CTL_PROTO(opt_hpa_slab_goal) +CTL_PROTO(opt_hpa_slab_max_alloc) CTL_PROTO(opt_metadata_thp) CTL_PROTO(opt_retain) CTL_PROTO(opt_dss) @@ -345,6 +347,8 @@ static const ctl_named_node_t opt_node[] = { {NAME("abort_conf"), CTL(opt_abort_conf)}, {NAME("confirm_conf"), CTL(opt_confirm_conf)}, {NAME("hpa"), CTL(opt_hpa)}, + {NAME("hpa_slab_goal"), CTL(opt_hpa_slab_goal)}, + {NAME("hpa_max_alloc"), CTL(opt_hpa_slab_max_alloc)}, {NAME("metadata_thp"), CTL(opt_metadata_thp)}, {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)}, @@ -1819,6 +1823,8 @@ CTL_RO_NL_GEN(opt_abort, opt_abort, bool) CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool) +CTL_RO_NL_GEN(opt_hpa_slab_goal, opt_hpa_slab_goal, size_t) +CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_slab_max_alloc, size_t) CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *) CTL_RO_NL_GEN(opt_retain, opt_retain, bool) diff --git a/src/jemalloc.c b/src/jemalloc.c index 0dc685b2..fd822e07 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -136,6 +136,8 @@ malloc_mutex_t arenas_lock; /* The global hpa, and whether it's on. */ bool opt_hpa = false; +size_t opt_hpa_slab_goal = 512 * 1024; +size_t opt_hpa_slab_max_alloc = 256 * 1024; /* * Arenas that are used to service external requests. Not all elements of the @@ -1481,6 +1483,17 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_CHECK_MIN, CONF_CHECK_MAX, true); CONF_HANDLE_BOOL(opt_hpa, "hpa") + /* + * If someone violates these mins and maxes, they're + * confused. + */ + CONF_HANDLE_SIZE_T(opt_hpa_slab_goal, "hpa_slab_goal", + PAGE, 512 * PAGE, CONF_CHECK_MIN, CONF_CHECK_MAX, + true) + CONF_HANDLE_SIZE_T(opt_hpa_slab_max_alloc, + "hpa_slab_max_alloc", PAGE, 512 * PAGE, + CONF_CHECK_MIN, CONF_CHECK_MAX, true) + if (CONF_MATCH("slab_sizes")) { if (CONF_MATCH_VALUE("default")) { sc_data_init(sc_data); @@ -1787,7 +1800,8 @@ malloc_init_hard_a0_locked() { &a0->pa_shard.edata_cache)) { return true; } - if (pa_shard_enable_hpa(&a0->pa_shard, &arena_hpa_global)) { + if (pa_shard_enable_hpa(&a0->pa_shard, &arena_hpa_global, + opt_hpa_slab_goal, opt_hpa_slab_max_alloc)) { return true; } } diff --git a/src/pa.c b/src/pa.c index 672db7b0..a8aa32d3 100644 --- a/src/pa.c +++ b/src/pa.c @@ -48,13 +48,14 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, } bool -pa_shard_enable_hpa(pa_shard_t *shard, hpa_t *hpa) { - /* - * These are constants for now; eventually they'll probably be - * tuneable. - */ - size_t ps_goal = 512 * 1024; - size_t ps_alloc_max = 256 * 1024; +pa_shard_enable_hpa(pa_shard_t *shard, hpa_t *hpa, size_t ps_goal, + size_t ps_alloc_max) { + ps_goal &= ~PAGE_MASK; + ps_alloc_max &= ~PAGE_MASK; + + if (ps_alloc_max > ps_goal) { + ps_alloc_max = ps_goal; + } if (hpa_shard_init(&shard->hpa_shard, hpa, &shard->edata_cache, shard->ind, ps_goal, ps_alloc_max)) { return true; diff --git a/src/stats.c b/src/stats.c index b2ec57b7..78068f49 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1096,6 +1096,8 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_CHAR_P("percpu_arena") OPT_WRITE_SIZE_T("oversize_threshold") OPT_WRITE_BOOL("hpa") + OPT_WRITE_SIZE_T("hpa_slab_goal") + OPT_WRITE_SIZE_T("hpa_slab_max_alloc") OPT_WRITE_CHAR_P("metadata_thp") OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread") OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms")