HPA: Make slab sizes and maxes configurable.

This allows easy experimentation with them as tuning parameters.

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -13,6 +13,8 @@ extern bool opt_abort;
 extern bool opt_abort_conf;
 extern bool opt_confirm_conf;
 extern bool opt_hpa;
+extern size_t opt_hpa_slab_goal;
+extern size_t opt_hpa_slab_max_alloc;
 extern const char *opt_junk;
 extern bool opt_junk_alloc;
 extern bool opt_junk_free;

include/jemalloc/internal/pa.h

@@ -123,7 +123,8 @@ bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
  * This isn't exposed to users; we allow late enablement of the HPA shard so
  * that we can boot without worrying about the HPA, then turn it on in a0.
  */
-bool pa_shard_enable_hpa(pa_shard_t *shard, hpa_t *hpa);
+bool pa_shard_enable_hpa(pa_shard_t *shard, hpa_t *hpa, size_t ps_goal,
+    size_t ps_alloc_max);
 /*
  * We stop using the HPA when custom extent hooks are installed, but still
  * redirect deallocations to it.

src/arena.c

@@ -1527,7 +1527,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 	 *   so arena_hpa_global is not yet initialized.
 	 */
 	if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) {
-		if (pa_shard_enable_hpa(&arena->pa_shard, &arena_hpa_global)) {
+		if (pa_shard_enable_hpa(&arena->pa_shard, &arena_hpa_global,
+		    opt_hpa_slab_goal, opt_hpa_slab_max_alloc)) {
 			goto label_error;
 		}
 	}

src/ctl.c

@@ -91,6 +91,8 @@ CTL_PROTO(opt_abort)
 CTL_PROTO(opt_abort_conf)
 CTL_PROTO(opt_confirm_conf)
 CTL_PROTO(opt_hpa)
+CTL_PROTO(opt_hpa_slab_goal)
+CTL_PROTO(opt_hpa_slab_max_alloc)
 CTL_PROTO(opt_metadata_thp)
 CTL_PROTO(opt_retain)
 CTL_PROTO(opt_dss)
@@ -345,6 +347,8 @@ static const ctl_named_node_t opt_node[] = {
 	{NAME("abort_conf"),	CTL(opt_abort_conf)},
 	{NAME("confirm_conf"),	CTL(opt_confirm_conf)},
 	{NAME("hpa"),		CTL(opt_hpa)},
+	{NAME("hpa_slab_goal"),	CTL(opt_hpa_slab_goal)},
+	{NAME("hpa_max_alloc"),	CTL(opt_hpa_slab_max_alloc)},
 	{NAME("metadata_thp"),	CTL(opt_metadata_thp)},
 	{NAME("retain"),	CTL(opt_retain)},
 	{NAME("dss"),		CTL(opt_dss)},
@@ -1819,6 +1823,8 @@ CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
 CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool)
 CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool)
 CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool)
+CTL_RO_NL_GEN(opt_hpa_slab_goal, opt_hpa_slab_goal, size_t)
+CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_slab_max_alloc, size_t)
 CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp],
     const char *)
 CTL_RO_NL_GEN(opt_retain, opt_retain, bool)

src/jemalloc.c

@@ -136,6 +136,8 @@ malloc_mutex_t arenas_lock;
 
 /* The global hpa, and whether it's on. */
 bool opt_hpa = false;
+size_t opt_hpa_slab_goal = 512 * 1024;
+size_t opt_hpa_slab_max_alloc = 256 * 1024;
 
 /*
  * Arenas that are used to service external requests.  Not all elements of the
@@ -1481,6 +1483,17 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 					   CONF_CHECK_MIN, CONF_CHECK_MAX,
 					   true);
 			CONF_HANDLE_BOOL(opt_hpa, "hpa")
+			/*
+			 * If someone violates these mins and maxes, they're
+			 * confused.
+			 */
+			CONF_HANDLE_SIZE_T(opt_hpa_slab_goal, "hpa_slab_goal",
+			    PAGE, 512 * PAGE, CONF_CHECK_MIN, CONF_CHECK_MAX,
+			    true)
+			CONF_HANDLE_SIZE_T(opt_hpa_slab_max_alloc,
+			    "hpa_slab_max_alloc", PAGE, 512 * PAGE,
+			    CONF_CHECK_MIN, CONF_CHECK_MAX, true)
+
 			if (CONF_MATCH("slab_sizes")) {
 				if (CONF_MATCH_VALUE("default")) {
 					sc_data_init(sc_data);
@@ -1787,7 +1800,8 @@ malloc_init_hard_a0_locked() {
 		    &a0->pa_shard.edata_cache)) {
 			return true;
 		}
-		if (pa_shard_enable_hpa(&a0->pa_shard, &arena_hpa_global)) {
+		if (pa_shard_enable_hpa(&a0->pa_shard, &arena_hpa_global,
+		    opt_hpa_slab_goal, opt_hpa_slab_max_alloc)) {
 			return true;
 		}
 	}

src/pa.c

@@ -48,13 +48,14 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
 }
 
 bool
-pa_shard_enable_hpa(pa_shard_t *shard, hpa_t *hpa) {
-	/*
-	 * These are constants for now; eventually they'll probably be
-	 * tuneable.
-	 */
-	size_t ps_goal = 512 * 1024;
-	size_t ps_alloc_max = 256 * 1024;
+pa_shard_enable_hpa(pa_shard_t *shard, hpa_t *hpa, size_t ps_goal,
+    size_t ps_alloc_max) {
+	ps_goal &= ~PAGE_MASK;
+	ps_alloc_max &= ~PAGE_MASK;
+
+	if (ps_alloc_max > ps_goal) {
+		ps_alloc_max = ps_goal;
+	}
 	if (hpa_shard_init(&shard->hpa_shard, hpa, &shard->edata_cache,
 	    shard->ind, ps_goal, ps_alloc_max)) {
 		return true;

src/stats.c

@@ -1096,6 +1096,8 @@ stats_general_print(emitter_t *emitter) {
 	OPT_WRITE_CHAR_P("percpu_arena")
 	OPT_WRITE_SIZE_T("oversize_threshold")
 	OPT_WRITE_BOOL("hpa")
+	OPT_WRITE_SIZE_T("hpa_slab_goal")
+	OPT_WRITE_SIZE_T("hpa_slab_max_alloc")
 	OPT_WRITE_CHAR_P("metadata_thp")
 	OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread")
 	OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms")