From 4790db15ed2bc751f1b96404358a42bd50c8a461 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Wed, 9 Dec 2020 13:52:29 -0800 Subject: [PATCH] HPA: make the hugification threshold configurable. --- include/jemalloc/internal/hpa_opts.h | 9 ++++++++- src/ctl.c | 5 +++++ src/hpa.c | 8 +++++--- src/jemalloc.c | 23 +++++++++++++++++++++++ src/stats.c | 1 + 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/include/jemalloc/internal/hpa_opts.h b/include/jemalloc/internal/hpa_opts.h index 95e86b46..bce0de2c 100644 --- a/include/jemalloc/internal/hpa_opts.h +++ b/include/jemalloc/internal/hpa_opts.h @@ -15,11 +15,18 @@ struct hpa_shard_opts_s { * any allocation request. */ size_t slab_max_alloc; + /* + * When the number of active bytes in a hugepage is >= + * hugification_threshold, we force hugify it. + */ + size_t hugification_threshold; }; #define HPA_SHARD_OPTS_DEFAULT { \ /* slab_max_alloc */ \ - 64 * 1024 \ + 64 * 1024, \ + /* hugification_threshold */ \ + HUGEPAGE * 95 / 100, \ } #endif /* JEMALLOC_INTERNAL_HPA_OPTS_H */ diff --git a/src/ctl.c b/src/ctl.c index 195a46e9..5096162c 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -94,6 +94,7 @@ CTL_PROTO(opt_trust_madvise) CTL_PROTO(opt_confirm_conf) CTL_PROTO(opt_hpa) CTL_PROTO(opt_hpa_slab_max_alloc) +CTL_PROTO(opt_hpa_hugification_threshold) CTL_PROTO(opt_hpa_sec_max_alloc) CTL_PROTO(opt_hpa_sec_max_bytes) CTL_PROTO(opt_hpa_sec_nshards) @@ -396,6 +397,8 @@ static const ctl_named_node_t opt_node[] = { {NAME("confirm_conf"), CTL(opt_confirm_conf)}, {NAME("hpa"), CTL(opt_hpa)}, {NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)}, + {NAME("hpa_hugification_threshold"), + CTL(opt_hpa_hugification_threshold)}, {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)}, {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)}, {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, @@ -2091,6 +2094,8 @@ CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool) CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool) CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t) +CTL_RO_NL_GEN(opt_hpa_hugification_threshold, + opt_hpa_opts.hugification_threshold, size_t) CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_max_alloc, size_t) CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_max_bytes, size_t) CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_nshards, size_t) diff --git a/src/hpa.c b/src/hpa.c index dd9be5ad..00fb279d 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -137,10 +137,12 @@ hpa_alloc_ps(tsdn_t *tsdn, hpa_shard_t *shard) { static bool hpa_good_hugification_candidate(hpa_shard_t *shard, hpdata_t *ps) { /* - * For now, just use a static check; hugify a page if it's <= 5% - * inactive. Eventually, this should be a malloc conf option. + * Note that this needs to be >= rather than just >, because of the + * important special case in which the hugification threshold is exactly + * HUGEPAGE. */ - return hpdata_nactive_get(ps) >= (HUGEPAGE_PAGES) * 95 / 100; + return hpdata_nactive_get(ps) * PAGE + >= shard->opts.hugification_threshold; } static bool diff --git a/src/jemalloc.c b/src/jemalloc.c index d1b09dd2..cd402621 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1414,6 +1414,29 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], "hpa_slab_max_alloc", PAGE, HUGEPAGE, CONF_CHECK_MIN, CONF_CHECK_MAX, true); + /* + * Accept either a ratio-based or an exact hugification + * threshold. + */ + CONF_HANDLE_SIZE_T(opt_hpa_opts.hugification_threshold, + "hpa_hugification_threshold", PAGE, HUGEPAGE, + CONF_CHECK_MIN, CONF_CHECK_MAX, true); + if (CONF_MATCH("hpa_hugification_threshold_ratio")) { + fxp_t ratio; + char *end; + bool err = fxp_parse(&ratio, v, + &end); + if (err || (size_t)(end - v) != vlen + || ratio > FXP_INIT_INT(1)) { + CONF_ERROR("Invalid conf value", + k, klen, v, vlen); + } else { + opt_hpa_opts.hugification_threshold = + fxp_mul_frac(HUGEPAGE, ratio); + } + CONF_CONTINUE; + } + CONF_HANDLE_SIZE_T(opt_hpa_sec_max_alloc, "hpa_sec_max_alloc", PAGE, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); CONF_HANDLE_SIZE_T(opt_hpa_sec_max_bytes, "hpa_sec_max_bytes", diff --git a/src/stats.c b/src/stats.c index 8e29656e..27fe5b76 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1464,6 +1464,7 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_SIZE_T("oversize_threshold") OPT_WRITE_BOOL("hpa") OPT_WRITE_SIZE_T("hpa_slab_max_alloc") + OPT_WRITE_SIZE_T("hpa_hugification_threshold") OPT_WRITE_SIZE_T("hpa_sec_max_alloc") OPT_WRITE_SIZE_T("hpa_sec_max_bytes") OPT_WRITE_SIZE_T("hpa_sec_nshards")