HPA: Introduce a redesigned hpa_central_t.

For now, this only handles allocating virtual address space to shards, with no
reuse.  This is framework, though; it will change over time.

include/jemalloc/internal/arena_externs.h

@@ -99,7 +99,7 @@ bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
     unsigned *binshard);
 size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
     void **ptrs, size_t nfill, bool zero);
-void arena_boot(sc_data_t *sc_data);
+bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
 void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
 void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
 void arena_prefork2(tsdn_t *tsdn, arena_t *arena);

include/jemalloc/internal/hpa.h

@@ -7,6 +7,37 @@
 #include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/psset.h"
 
+typedef struct hpa_central_s hpa_central_t;
+struct hpa_central_s {
+	/*
+	 * The mutex guarding most of the operations on the central data
+	 * structure.
+	 */
+	malloc_mutex_t mtx;
+	/*
+	 * Guards expansion of eden.  We separate this from the regular mutex so
+	 * that cheaper operations can still continue while we're doing the OS
+	 * call.
+	 */
+	malloc_mutex_t grow_mtx;
+	/*
+	 * Either NULL (if empty), or some integer multiple of a
+	 * hugepage-aligned number of hugepages.  We carve them off one at a
+	 * time to satisfy new pageslab requests.
+	 *
+	 * Guarded by grow_mtx.
+	 */
+	void *eden;
+	size_t eden_len;
+	/* Source for metadata. */
+	base_t *base;
+	/* Number of grow operations done on this hpa_central_t. */
+	uint64_t age_counter;
+
+	/* The HPA hooks. */
+	hpa_hooks_t hooks;
+};
+
 typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
 struct hpa_shard_nonderived_stats_s {
 	/*
@@ -52,19 +83,20 @@ struct hpa_shard_s {
 	 * pointer to the hpa_shard_t.
 	 */
 	pai_t pai;
-	malloc_mutex_t grow_mtx;
+
+	/* The central allocator we get our hugepages from. */
+	hpa_central_t *central;
+	/* Protects most of this shard's state. */
 	malloc_mutex_t mtx;
+	/*
+	 * Guards the shard's access to the central allocator (preventing
+	 * multiple threads operating on this shard from accessing the central
+	 * allocator).
+	 */
+	malloc_mutex_t grow_mtx;
 	/* The base metadata allocator. */
 	base_t *base;
 
-	/*
-	 * The HPA hooks for this shard.  Eventually, once we have the
-	 * hpa_central_t back, these should live there (since it doesn't make
-	 * sense for different shards on the same hpa_central_t to have
-	 * different hooks).
-	 */
-	hpa_hooks_t hooks;
-
 	/*
 	 * This edata cache is the one we use when allocating a small extent
 	 * from a pageslab.  The pageslab itself comes from the centralized
@@ -81,18 +113,13 @@ struct hpa_shard_s {
 	 */
 	uint64_t age_counter;
 
-	/*
-	 * Either NULL (if empty), or some integer multiple of a
-	 * hugepage-aligned number of hugepages.  We carve them off one at a
-	 * time to satisfy new pageslab requests.
-	 *
-	 * Guarded by grow_mtx.
-	 */
-	void *eden;
-	size_t eden_len;
-
 	/* The arena ind we're associated with. */
 	unsigned ind;
+
+	/*
+	 * Our emap.  This is just a cache of the emap pointer in the associated
+	 * hpa_central.
+	 */
 	emap_t *emap;
 
 	/* The configuration choices for this hpa shard. */
@@ -117,8 +144,9 @@ struct hpa_shard_s {
  * just that it can function properly given the system it's running on.
  */
 bool hpa_supported();
-bool hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
+bool hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
-    edata_cache_t *edata_cache, unsigned ind, const hpa_hooks_t *hooks,
+bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
+    base_t *base, edata_cache_t *edata_cache, unsigned ind,
     const hpa_shard_opts_t *opts);
 
 void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);

include/jemalloc/internal/pa.h

@@ -20,6 +20,11 @@
  * others will be coming soon.
  */
 
+typedef struct pa_central_s pa_central_t;
+struct pa_central_s {
+	hpa_central_t hpa;
+};
+
 /*
  * The stats for a particular pa_shard.  Because of the way the ctl module
  * handles stats epoch data collection (it has its own arena_stats, and merges
@@ -61,6 +66,9 @@ struct pa_shard_stats_s {
  */
 typedef struct pa_shard_s pa_shard_t;
 struct pa_shard_s {
+	/* The central PA this shard is associated with. */
+	pa_central_t *central;
+
 	/*
 	 * Number of pages in active extents.
 	 *
@@ -76,6 +84,7 @@ struct pa_shard_s {
 	 * for those allocations.
 	 */
 	atomic_b_t use_hpa;
+
 	/*
 	 * If we never used the HPA to begin with, it wasn't initialized, and so
 	 * we shouldn't try to e.g. acquire its mutexes during fork.  This
@@ -121,18 +130,21 @@ pa_shard_ehooks_get(pa_shard_t *shard) {
 }
 
 /* Returns true on error. */
-bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
+bool pa_central_init(pa_central_t *central, base_t *base, bool hpa,
-    unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
+    hpa_hooks_t *hpa_hooks);
-    nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
+
-    ssize_t muzzy_decay_ms);
+/* Returns true on error. */
+bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
+    emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats,
+    malloc_mutex_t *stats_mtx, nstime_t *cur_time, size_t oversize_threshold,
+    ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms);
 
 /*
  * This isn't exposed to users; we allow late enablement of the HPA shard so
  * that we can boot without worrying about the HPA, then turn it on in a0.
  */
 bool pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
-    const hpa_hooks_t *hpa_hooks, const hpa_shard_opts_t *hpa_opts,
+    const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts);
-    const sec_opts_t *hpa_sec_opts);
 
 /*
  * We stop using the HPA when custom extent hooks are installed, but still

include/jemalloc/internal/witness.h

@@ -52,8 +52,8 @@ enum witness_rank_e {
 	WITNESS_RANK_EXTENTS,
 	WITNESS_RANK_HPA_SHARD = WITNESS_RANK_EXTENTS,
 
-	WITNESS_RANK_HPA_GROW,
+	WITNESS_RANK_HPA_CENTRAL_GROW,
-	WITNESS_RANK_HPA,
+	WITNESS_RANK_HPA_CENTRAL,
 
 	WITNESS_RANK_EDATA_CACHE,
 

src/arena.c

@@ -36,6 +36,7 @@ static atomic_zd_t dirty_decay_ms_default;
 static atomic_zd_t muzzy_decay_ms_default;
 
 emap_t arena_emap_global;
+pa_central_t arena_pa_central_global;
 
 const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = {
 #define STEP(step, h, x, y)			\
@@ -1541,9 +1542,10 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 
 	nstime_t cur_time;
 	nstime_init_update(&cur_time);
-	if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind,
+	if (pa_shard_init(tsdn, &arena->pa_shard, &arena_pa_central_global,
-	    &arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx),
+	    &arena_emap_global, base, ind, &arena->stats.pa_shard_stats,
-	    &cur_time, oversize_threshold, arena_dirty_decay_ms_default_get(),
+	    LOCKEDINT_MTX(arena->stats.mtx), &cur_time, oversize_threshold,
+	    arena_dirty_decay_ms_default_get(),
 	    arena_muzzy_decay_ms_default_get())) {
 		goto label_error;
 	}
@@ -1575,7 +1577,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 		hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
 		hpa_shard_opts.deferral_allowed = background_thread_enabled();
 		if (pa_shard_enable_hpa(tsdn, &arena->pa_shard,
-		    &hpa_hooks_default, &hpa_shard_opts, &opt_hpa_sec_opts)) {
+		    &hpa_shard_opts, &opt_hpa_sec_opts)) {
 			goto label_error;
 		}
 	}
@@ -1664,8 +1666,8 @@ arena_is_huge(unsigned arena_ind) {
 	return (arena_ind == huge_arena_ind);
 }
 
-void
+bool
-arena_boot(sc_data_t *sc_data) {
+arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) {
 	arena_dirty_decay_ms_default_set(opt_dirty_decay_ms);
 	arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms);
 	for (unsigned i = 0; i < SC_NBINS; i++) {
@@ -1680,6 +1682,8 @@ arena_boot(sc_data_t *sc_data) {
 		nbins_total += bin_infos[i].n_shards;
 		cur_offset += (uint32_t)(bin_infos[i].n_shards * sizeof(bin_t));
 	}
+	return pa_central_init(&arena_pa_central_global, base, hpa,
+	    &hpa_hooks_default);
 }
 
 void

src/hpa.c

@@ -51,9 +51,125 @@ hpa_supported() {
 }
 
 bool
-hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
+hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) {
-    edata_cache_t *edata_cache, unsigned ind,
+	/* malloc_conf processing should have filtered out these cases. */
-    const hpa_hooks_t *hooks, const hpa_shard_opts_t *opts) {
+	assert(hpa_supported());
+	bool err;
+	err = malloc_mutex_init(&central->grow_mtx, "hpa_central_grow",
+	    WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive);
+	if (err) {
+		return true;
+	}
+	err = malloc_mutex_init(&central->mtx, "hpa_central",
+	    WITNESS_RANK_HPA_CENTRAL, malloc_mutex_rank_exclusive);
+	if (err) {
+		return true;
+	}
+	central->base = base;
+	central->eden = NULL;
+	central->eden_len = 0;
+	central->age_counter = 0;
+	central->hooks = *hooks;
+	return false;
+}
+
+static hpdata_t *
+hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) {
+	return (hpdata_t *)base_alloc(tsdn, central->base, sizeof(hpdata_t),
+	    CACHELINE);
+}
+
+hpdata_t *
+hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
+    bool *oom) {
+	/* Don't yet support big allocations; these should get filtered out. */
+	assert(size <= HUGEPAGE);
+	/*
+	 * Should only try to extract from the central allocator if the local
+	 * shard is exhausted.  We should hold the grow_mtx on that shard.
+	 */
+	witness_assert_positive_depth_to_rank(
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW);
+
+	malloc_mutex_lock(tsdn, &central->grow_mtx);
+	*oom = false;
+
+	hpdata_t *ps = NULL;
+
+	/* Is eden a perfect fit? */
+	if (central->eden != NULL && central->eden_len == HUGEPAGE) {
+		ps = hpa_alloc_ps(tsdn, central);
+		if (ps == NULL) {
+			*oom = true;
+			malloc_mutex_unlock(tsdn, &central->grow_mtx);
+			return NULL;
+		}
+		hpdata_init(ps, central->eden, central->age_counter++);
+		central->eden = NULL;
+		central->eden_len = 0;
+		malloc_mutex_unlock(tsdn, &central->grow_mtx);
+		return ps;
+	}
+
+	/*
+	 * We're about to try to allocate from eden by splitting.  If eden is
+	 * NULL, we have to allocate it too.  Otherwise, we just have to
+	 * allocate an edata_t for the new psset.
+	 */
+	if (central->eden == NULL) {
+		/*
+		 * During development, we're primarily concerned with systems
+		 * with overcommit.  Eventually, we should be more careful here.
+		 */
+		bool commit = true;
+		/* Allocate address space, bailing if we fail. */
+		void *new_eden = pages_map(NULL, HPA_EDEN_SIZE, HUGEPAGE,
+		    &commit);
+		if (new_eden == NULL) {
+			*oom = true;
+			malloc_mutex_unlock(tsdn, &central->grow_mtx);
+			return NULL;
+		}
+		ps = hpa_alloc_ps(tsdn, central);
+		if (ps == NULL) {
+			pages_unmap(new_eden, HPA_EDEN_SIZE);
+			*oom = true;
+			malloc_mutex_unlock(tsdn, &central->grow_mtx);
+			return NULL;
+		}
+		central->eden = new_eden;
+		central->eden_len = HPA_EDEN_SIZE;
+	} else {
+		/* Eden is already nonempty; only need an edata for ps. */
+		ps = hpa_alloc_ps(tsdn, central);
+		if (ps == NULL) {
+			*oom = true;
+			malloc_mutex_unlock(tsdn, &central->grow_mtx);
+			return NULL;
+		}
+	}
+	assert(ps != NULL);
+	assert(central->eden != NULL);
+	assert(central->eden_len > HUGEPAGE);
+	assert(central->eden_len % HUGEPAGE == 0);
+	assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
+
+	hpdata_init(ps, central->eden, central->age_counter++);
+
+	char *eden_char = (char *)central->eden;
+	eden_char += HUGEPAGE;
+	central->eden = (void *)eden_char;
+	central->eden_len -= HUGEPAGE;
+
+	malloc_mutex_unlock(tsdn, &central->grow_mtx);
+
+	return ps;
+}
+
+bool
+hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
+    base_t *base, edata_cache_t *edata_cache, unsigned ind,
+    const hpa_shard_opts_t *opts) {
 	/* malloc_conf processing should have filtered out these cases. */
 	assert(hpa_supported());
 	bool err;
@@ -69,13 +185,11 @@ hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
 	}
 
 	assert(edata_cache != NULL);
+	shard->central = central;
 	shard->base = base;
-	shard->hooks = *hooks;
 	edata_cache_small_init(&shard->ecs, edata_cache);
 	psset_init(&shard->psset);
 	shard->age_counter = 0;
-	shard->eden = NULL;
-	shard->eden_len = 0;
 	shard->ind = ind;
 	shard->emap = emap;
 
@@ -136,12 +250,6 @@ hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
 	malloc_mutex_unlock(tsdn, &shard->grow_mtx);
 }
 
-static hpdata_t *
-hpa_alloc_ps(tsdn_t *tsdn, hpa_shard_t *shard) {
-	return (hpdata_t *)base_alloc(tsdn, shard->base, sizeof(hpdata_t),
-	    CACHELINE);
-}
-
 static bool
 hpa_good_hugification_candidate(hpa_shard_t *shard, hpdata_t *ps) {
 	/*
@@ -227,7 +335,7 @@ hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard,
 	if (hpa_good_hugification_candidate(shard, ps)
 	    && !hpdata_huge_get(ps)) {
 		nstime_t now;
-		shard->hooks.curtime(&now);
+		shard->central->hooks.curtime(&now);
 		hpdata_allow_hugify(ps, now);
 	}
 	/*
@@ -247,64 +355,6 @@ hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard,
 	}
 }
 
-static hpdata_t *
-hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) {
-	malloc_mutex_assert_owner(tsdn, &shard->grow_mtx);
-	hpdata_t *ps = NULL;
-
-	/* Is eden a perfect fit? */
-	if (shard->eden != NULL && shard->eden_len == HUGEPAGE) {
-		ps = hpa_alloc_ps(tsdn, shard);
-		if (ps == NULL) {
-			return NULL;
-		}
-		hpdata_init(ps, shard->eden, shard->age_counter++);
-		shard->eden = NULL;
-		shard->eden_len = 0;
-		return ps;
-	}
-
-	/*
-	 * We're about to try to allocate from eden by splitting.  If eden is
-	 * NULL, we have to allocate it too.  Otherwise, we just have to
-	 * allocate an edata_t for the new psset.
-	 */
-	if (shard->eden == NULL) {
-		/* Allocate address space, bailing if we fail. */
-		void *new_eden = shard->hooks.map(HPA_EDEN_SIZE);
-		if (new_eden == NULL) {
-			return NULL;
-		}
-		ps = hpa_alloc_ps(tsdn, shard);
-		if (ps == NULL) {
-			shard->hooks.unmap(new_eden, HPA_EDEN_SIZE);
-			return NULL;
-		}
-		shard->eden = new_eden;
-		shard->eden_len = HPA_EDEN_SIZE;
-	} else {
-		/* Eden is already nonempty; only need an edata for ps. */
-		ps = hpa_alloc_ps(tsdn, shard);
-		if (ps == NULL) {
-			return NULL;
-		}
-	}
-	assert(ps != NULL);
-	assert(shard->eden != NULL);
-	assert(shard->eden_len > HUGEPAGE);
-	assert(shard->eden_len % HUGEPAGE == 0);
-	assert(HUGEPAGE_ADDR2BASE(shard->eden) == shard->eden);
-
-	hpdata_init(ps, shard->eden, shard->age_counter++);
-
-	char *eden_char = (char *)shard->eden;
-	eden_char += HUGEPAGE;
-	shard->eden = (void *)eden_char;
-	shard->eden_len -= HUGEPAGE;
-
-	return ps;
-}
-
 /* Returns whether or not we purged anything. */
 static bool
 hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
@@ -348,7 +398,8 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
 
 	/* Actually do the purging, now that the lock is dropped. */
 	if (dehugify) {
-		shard->hooks.dehugify(hpdata_addr_get(to_purge), HUGEPAGE);
+		shard->central->hooks.dehugify(hpdata_addr_get(to_purge),
+		    HUGEPAGE);
 	}
 	size_t total_purged = 0;
 	uint64_t purges_this_pass = 0;
@@ -359,7 +410,7 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
 		total_purged += purge_size;
 		assert(total_purged <= HUGEPAGE);
 		purges_this_pass++;
-		shard->hooks.purge(purge_addr, purge_size);
+		shard->central->hooks.purge(purge_addr, purge_size);
 	}
 
 	malloc_mutex_lock(tsdn, &shard->mtx);
@@ -406,7 +457,7 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) {
 	/* Make sure that it's been hugifiable for long enough. */
 	nstime_t time_hugify_allowed = hpdata_time_hugify_allowed(to_hugify);
 	nstime_t nstime;
-	shard->hooks.curtime(&nstime);
+	shard->central->hooks.curtime(&nstime);
 	nstime_subtract(&nstime, &time_hugify_allowed);
 	uint64_t millis = nstime_msec(&nstime);
 	if (millis < shard->opts.hugify_delay_ms) {
@@ -427,7 +478,7 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) {
 
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 
-	shard->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE);
+	shard->central->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE);
 
 	malloc_mutex_lock(tsdn, &shard->mtx);
 	shard->stats.nhugifies++;
@@ -604,7 +655,7 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
 	 * deallocations (and allocations of smaller sizes) may still succeed
 	 * while we're doing this potentially expensive system call.
 	 */
-	hpdata_t *ps = hpa_grow(tsdn, shard);
+	hpdata_t *ps = hpa_central_extract(tsdn, shard->central, size, &oom);
 	if (ps == NULL) {
 		malloc_mutex_unlock(tsdn, &shard->grow_mtx);
 		return nsuccess;
@@ -833,7 +884,7 @@ hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) {
 		/* There should be no allocations anywhere. */
 		assert(hpdata_empty(ps));
 		psset_remove(&shard->psset, ps);
-		shard->hooks.unmap(hpdata_addr_get(ps), HUGEPAGE);
+		shard->central->hooks.unmap(hpdata_addr_get(ps), HUGEPAGE);
 	}
 }
 

src/jemalloc.c

@@ -1747,7 +1747,19 @@ malloc_init_hard_a0_locked() {
 	if (config_prof) {
 		prof_boot1();
 	}
-	arena_boot(&sc_data);
+	if (opt_hpa && !hpa_supported()) {
+		malloc_printf("<jemalloc>: HPA not supported in the current "
+		    "configuration; %s.",
+		    opt_abort_conf ? "aborting" : "disabling");
+		if (opt_abort_conf) {
+			malloc_abort_invalid_conf();
+		} else {
+			opt_hpa = false;
+		}
+	}
+	if (arena_boot(&sc_data, b0get(), opt_hpa)) {
+		return true;
+	}
 	if (tcache_boot(TSDN_NULL, b0get())) {
 		return true;
 	}
@@ -1786,7 +1798,7 @@ malloc_init_hard_a0_locked() {
 		hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
 		hpa_shard_opts.deferral_allowed = background_thread_enabled();
 		if (pa_shard_enable_hpa(TSDN_NULL, &a0->pa_shard,
-		    &hpa_hooks_default, &hpa_shard_opts, &opt_hpa_sec_opts)) {
+		    &hpa_shard_opts, &opt_hpa_sec_opts)) {
 			return true;
 		}
 	}

src/pa.c

@@ -15,10 +15,23 @@ pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) {
 }
 
 bool
-pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
+pa_central_init(pa_central_t *central, base_t *base, bool hpa,
-    unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
+    hpa_hooks_t *hpa_hooks) {
-    nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
+	bool err;
-    ssize_t muzzy_decay_ms) {
+	if (hpa) {
+		err = hpa_central_init(&central->hpa, base, hpa_hooks);
+		if (err) {
+			return true;
+		}
+	}
+	return false;
+}
+
+bool
+pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
+    emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats,
+    malloc_mutex_t *stats_mtx, nstime_t *cur_time, size_t oversize_threshold,
+    ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
 	/* This will change eventually, but for now it should hold. */
 	assert(base_ind_get(base) == ind);
 	if (edata_cache_init(&shard->edata_cache, base)) {
@@ -42,6 +55,7 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
 	shard->stats = stats;
 	memset(shard->stats, 0, sizeof(*shard->stats));
 
+	shard->central = central;
 	shard->emap = emap;
 	shard->base = base;
 
@@ -50,10 +64,9 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
 
 bool
 pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
-    const hpa_hooks_t *hpa_hooks, const hpa_shard_opts_t *hpa_opts,
+    const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
-    const sec_opts_t *hpa_sec_opts) {
+	if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap,
-	if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base,
+	    shard->base, &shard->edata_cache, shard->ind, hpa_opts)) {
-	    &shard->edata_cache, shard->ind, hpa_hooks, hpa_opts)) {
 		return true;
 	}
 	if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai,

test/unit/hpa.c

@@ -13,6 +13,7 @@ struct test_data_s {
 	 * test_data_t and the hpa_shard_t;
 	 */
 	hpa_shard_t shard;
+	hpa_central_t central;
 	base_t *base;
 	edata_cache_t shard_edata_cache;
 
@@ -50,9 +51,12 @@ create_test_data(hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
 	err = emap_init(&test_data->emap, test_data->base, /* zeroed */ false);
 	assert_false(err, "");
 
-	err = hpa_shard_init(&test_data->shard, &test_data->emap,
+	err = hpa_central_init(&test_data->central, test_data->base, hooks);
-	    test_data->base, &test_data->shard_edata_cache, SHARD_IND,
+	assert_false(err, "");
-	    hooks, opts);
+
+	err = hpa_shard_init(&test_data->shard, &test_data->central,
+	    &test_data->emap, test_data->base, &test_data->shard_edata_cache,
+	    SHARD_IND, opts);
 	assert_false(err, "");
 
 	return (hpa_shard_t *)test_data;

test/unit/pa.c

@@ -40,6 +40,7 @@ init_test_extent_hooks(extent_hooks_t *hooks) {
 typedef struct test_data_s test_data_t;
 struct test_data_s {
 	pa_shard_t shard;
+	pa_central_t central;
 	base_t *base;
 	emap_t emap;
 	pa_shard_stats_t stats;
@@ -63,9 +64,13 @@ test_data_t *init_test_data(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
 	nstime_t time;
 	nstime_init(&time, 0);
 
+	err = pa_central_init(&test_data->central, base, opt_hpa,
+	    &hpa_hooks_default);
+	assert_false(err, "");
+
 	const size_t oversize_threshold = 8 * 1024 * 1024;
-	err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap,
+	err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->central,
-	    test_data->base, /* ind */ 1, &test_data->stats,
+	    &test_data->emap, test_data->base, /* ind */ 1, &test_data->stats,
 	    &test_data->stats_mtx, &time, oversize_threshold, dirty_decay_ms,
 	    muzzy_decay_ms);
 	assert_false(err, "");