From d93eef2f405b7c6e2a78f589a5037a26d4bd4d44 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Fri, 7 May 2021 13:54:26 -0700 Subject: [PATCH] HPA: Introduce a redesigned hpa_central_t. For now, this only handles allocating virtual address space to shards, with no reuse. This is framework, though; it will change over time. --- include/jemalloc/internal/arena_externs.h | 2 +- include/jemalloc/internal/hpa.h | 70 +++++--- include/jemalloc/internal/pa.h | 24 ++- include/jemalloc/internal/witness.h | 4 +- src/arena.c | 16 +- src/hpa.c | 205 ++++++++++++++-------- src/jemalloc.c | 16 +- src/pa.c | 29 ++- test/unit/hpa.c | 10 +- test/unit/pa.c | 9 +- 10 files changed, 257 insertions(+), 128 deletions(-) diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index bb3462f5..557e49f1 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -99,7 +99,7 @@ bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned *binshard); size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind, void **ptrs, size_t nfill, bool zero); -void arena_boot(sc_data_t *sc_data); +bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); diff --git a/include/jemalloc/internal/hpa.h b/include/jemalloc/internal/hpa.h index 3132a6f5..623f9c40 100644 --- a/include/jemalloc/internal/hpa.h +++ b/include/jemalloc/internal/hpa.h @@ -7,6 +7,37 @@ #include "jemalloc/internal/pai.h" #include "jemalloc/internal/psset.h" +typedef struct hpa_central_s hpa_central_t; +struct hpa_central_s { + /* + * The mutex guarding most of the operations on the central data + * structure. + */ + malloc_mutex_t mtx; + /* + * Guards expansion of eden. We separate this from the regular mutex so + * that cheaper operations can still continue while we're doing the OS + * call. + */ + malloc_mutex_t grow_mtx; + /* + * Either NULL (if empty), or some integer multiple of a + * hugepage-aligned number of hugepages. We carve them off one at a + * time to satisfy new pageslab requests. + * + * Guarded by grow_mtx. + */ + void *eden; + size_t eden_len; + /* Source for metadata. */ + base_t *base; + /* Number of grow operations done on this hpa_central_t. */ + uint64_t age_counter; + + /* The HPA hooks. */ + hpa_hooks_t hooks; +}; + typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t; struct hpa_shard_nonderived_stats_s { /* @@ -52,19 +83,20 @@ struct hpa_shard_s { * pointer to the hpa_shard_t. */ pai_t pai; - malloc_mutex_t grow_mtx; + + /* The central allocator we get our hugepages from. */ + hpa_central_t *central; + /* Protects most of this shard's state. */ malloc_mutex_t mtx; + /* + * Guards the shard's access to the central allocator (preventing + * multiple threads operating on this shard from accessing the central + * allocator). + */ + malloc_mutex_t grow_mtx; /* The base metadata allocator. */ base_t *base; - /* - * The HPA hooks for this shard. Eventually, once we have the - * hpa_central_t back, these should live there (since it doesn't make - * sense for different shards on the same hpa_central_t to have - * different hooks). - */ - hpa_hooks_t hooks; - /* * This edata cache is the one we use when allocating a small extent * from a pageslab. The pageslab itself comes from the centralized @@ -81,18 +113,13 @@ struct hpa_shard_s { */ uint64_t age_counter; - /* - * Either NULL (if empty), or some integer multiple of a - * hugepage-aligned number of hugepages. We carve them off one at a - * time to satisfy new pageslab requests. - * - * Guarded by grow_mtx. - */ - void *eden; - size_t eden_len; - /* The arena ind we're associated with. */ unsigned ind; + + /* + * Our emap. This is just a cache of the emap pointer in the associated + * hpa_central. + */ emap_t *emap; /* The configuration choices for this hpa shard. */ @@ -117,8 +144,9 @@ struct hpa_shard_s { * just that it can function properly given the system it's running on. */ bool hpa_supported(); -bool hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base, - edata_cache_t *edata_cache, unsigned ind, const hpa_hooks_t *hooks, +bool hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks); +bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, + base_t *base, edata_cache_t *edata_cache, unsigned ind, const hpa_shard_opts_t *opts); void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src); diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 582625b1..2e5b9ef0 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -20,6 +20,11 @@ * others will be coming soon. */ +typedef struct pa_central_s pa_central_t; +struct pa_central_s { + hpa_central_t hpa; +}; + /* * The stats for a particular pa_shard. Because of the way the ctl module * handles stats epoch data collection (it has its own arena_stats, and merges @@ -61,6 +66,9 @@ struct pa_shard_stats_s { */ typedef struct pa_shard_s pa_shard_t; struct pa_shard_s { + /* The central PA this shard is associated with. */ + pa_central_t *central; + /* * Number of pages in active extents. * @@ -76,6 +84,7 @@ struct pa_shard_s { * for those allocations. */ atomic_b_t use_hpa; + /* * If we never used the HPA to begin with, it wasn't initialized, and so * we shouldn't try to e.g. acquire its mutexes during fork. This @@ -121,18 +130,21 @@ pa_shard_ehooks_get(pa_shard_t *shard) { } /* Returns true on error. */ -bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, - unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx, - nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms, - ssize_t muzzy_decay_ms); +bool pa_central_init(pa_central_t *central, base_t *base, bool hpa, + hpa_hooks_t *hpa_hooks); + +/* Returns true on error. */ +bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central, + emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats, + malloc_mutex_t *stats_mtx, nstime_t *cur_time, size_t oversize_threshold, + ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms); /* * This isn't exposed to users; we allow late enablement of the HPA shard so * that we can boot without worrying about the HPA, then turn it on in a0. */ bool pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard, - const hpa_hooks_t *hpa_hooks, const hpa_shard_opts_t *hpa_opts, - const sec_opts_t *hpa_sec_opts); + const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts); /* * We stop using the HPA when custom extent hooks are installed, but still diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index 0c29321c..c12a705c 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -52,8 +52,8 @@ enum witness_rank_e { WITNESS_RANK_EXTENTS, WITNESS_RANK_HPA_SHARD = WITNESS_RANK_EXTENTS, - WITNESS_RANK_HPA_GROW, - WITNESS_RANK_HPA, + WITNESS_RANK_HPA_CENTRAL_GROW, + WITNESS_RANK_HPA_CENTRAL, WITNESS_RANK_EDATA_CACHE, diff --git a/src/arena.c b/src/arena.c index 5daeea31..a495ef64 100644 --- a/src/arena.c +++ b/src/arena.c @@ -36,6 +36,7 @@ static atomic_zd_t dirty_decay_ms_default; static atomic_zd_t muzzy_decay_ms_default; emap_t arena_emap_global; +pa_central_t arena_pa_central_global; const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = { #define STEP(step, h, x, y) \ @@ -1541,9 +1542,10 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { nstime_t cur_time; nstime_init_update(&cur_time); - if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind, - &arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx), - &cur_time, oversize_threshold, arena_dirty_decay_ms_default_get(), + if (pa_shard_init(tsdn, &arena->pa_shard, &arena_pa_central_global, + &arena_emap_global, base, ind, &arena->stats.pa_shard_stats, + LOCKEDINT_MTX(arena->stats.mtx), &cur_time, oversize_threshold, + arena_dirty_decay_ms_default_get(), arena_muzzy_decay_ms_default_get())) { goto label_error; } @@ -1575,7 +1577,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts; hpa_shard_opts.deferral_allowed = background_thread_enabled(); if (pa_shard_enable_hpa(tsdn, &arena->pa_shard, - &hpa_hooks_default, &hpa_shard_opts, &opt_hpa_sec_opts)) { + &hpa_shard_opts, &opt_hpa_sec_opts)) { goto label_error; } } @@ -1664,8 +1666,8 @@ arena_is_huge(unsigned arena_ind) { return (arena_ind == huge_arena_ind); } -void -arena_boot(sc_data_t *sc_data) { +bool +arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) { arena_dirty_decay_ms_default_set(opt_dirty_decay_ms); arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms); for (unsigned i = 0; i < SC_NBINS; i++) { @@ -1680,6 +1682,8 @@ arena_boot(sc_data_t *sc_data) { nbins_total += bin_infos[i].n_shards; cur_offset += (uint32_t)(bin_infos[i].n_shards * sizeof(bin_t)); } + return pa_central_init(&arena_pa_central_global, base, hpa, + &hpa_hooks_default); } void diff --git a/src/hpa.c b/src/hpa.c index 4ae30b97..10594587 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -51,9 +51,125 @@ hpa_supported() { } bool -hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base, - edata_cache_t *edata_cache, unsigned ind, - const hpa_hooks_t *hooks, const hpa_shard_opts_t *opts) { +hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) { + /* malloc_conf processing should have filtered out these cases. */ + assert(hpa_supported()); + bool err; + err = malloc_mutex_init(¢ral->grow_mtx, "hpa_central_grow", + WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive); + if (err) { + return true; + } + err = malloc_mutex_init(¢ral->mtx, "hpa_central", + WITNESS_RANK_HPA_CENTRAL, malloc_mutex_rank_exclusive); + if (err) { + return true; + } + central->base = base; + central->eden = NULL; + central->eden_len = 0; + central->age_counter = 0; + central->hooks = *hooks; + return false; +} + +static hpdata_t * +hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) { + return (hpdata_t *)base_alloc(tsdn, central->base, sizeof(hpdata_t), + CACHELINE); +} + +hpdata_t * +hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size, + bool *oom) { + /* Don't yet support big allocations; these should get filtered out. */ + assert(size <= HUGEPAGE); + /* + * Should only try to extract from the central allocator if the local + * shard is exhausted. We should hold the grow_mtx on that shard. + */ + witness_assert_positive_depth_to_rank( + tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW); + + malloc_mutex_lock(tsdn, ¢ral->grow_mtx); + *oom = false; + + hpdata_t *ps = NULL; + + /* Is eden a perfect fit? */ + if (central->eden != NULL && central->eden_len == HUGEPAGE) { + ps = hpa_alloc_ps(tsdn, central); + if (ps == NULL) { + *oom = true; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return NULL; + } + hpdata_init(ps, central->eden, central->age_counter++); + central->eden = NULL; + central->eden_len = 0; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return ps; + } + + /* + * We're about to try to allocate from eden by splitting. If eden is + * NULL, we have to allocate it too. Otherwise, we just have to + * allocate an edata_t for the new psset. + */ + if (central->eden == NULL) { + /* + * During development, we're primarily concerned with systems + * with overcommit. Eventually, we should be more careful here. + */ + bool commit = true; + /* Allocate address space, bailing if we fail. */ + void *new_eden = pages_map(NULL, HPA_EDEN_SIZE, HUGEPAGE, + &commit); + if (new_eden == NULL) { + *oom = true; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return NULL; + } + ps = hpa_alloc_ps(tsdn, central); + if (ps == NULL) { + pages_unmap(new_eden, HPA_EDEN_SIZE); + *oom = true; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return NULL; + } + central->eden = new_eden; + central->eden_len = HPA_EDEN_SIZE; + } else { + /* Eden is already nonempty; only need an edata for ps. */ + ps = hpa_alloc_ps(tsdn, central); + if (ps == NULL) { + *oom = true; + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + return NULL; + } + } + assert(ps != NULL); + assert(central->eden != NULL); + assert(central->eden_len > HUGEPAGE); + assert(central->eden_len % HUGEPAGE == 0); + assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden); + + hpdata_init(ps, central->eden, central->age_counter++); + + char *eden_char = (char *)central->eden; + eden_char += HUGEPAGE; + central->eden = (void *)eden_char; + central->eden_len -= HUGEPAGE; + + malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); + + return ps; +} + +bool +hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, + base_t *base, edata_cache_t *edata_cache, unsigned ind, + const hpa_shard_opts_t *opts) { /* malloc_conf processing should have filtered out these cases. */ assert(hpa_supported()); bool err; @@ -69,13 +185,11 @@ hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base, } assert(edata_cache != NULL); + shard->central = central; shard->base = base; - shard->hooks = *hooks; edata_cache_small_init(&shard->ecs, edata_cache); psset_init(&shard->psset); shard->age_counter = 0; - shard->eden = NULL; - shard->eden_len = 0; shard->ind = ind; shard->emap = emap; @@ -136,12 +250,6 @@ hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard, malloc_mutex_unlock(tsdn, &shard->grow_mtx); } -static hpdata_t * -hpa_alloc_ps(tsdn_t *tsdn, hpa_shard_t *shard) { - return (hpdata_t *)base_alloc(tsdn, shard->base, sizeof(hpdata_t), - CACHELINE); -} - static bool hpa_good_hugification_candidate(hpa_shard_t *shard, hpdata_t *ps) { /* @@ -227,7 +335,7 @@ hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard, if (hpa_good_hugification_candidate(shard, ps) && !hpdata_huge_get(ps)) { nstime_t now; - shard->hooks.curtime(&now); + shard->central->hooks.curtime(&now); hpdata_allow_hugify(ps, now); } /* @@ -247,64 +355,6 @@ hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard, } } -static hpdata_t * -hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) { - malloc_mutex_assert_owner(tsdn, &shard->grow_mtx); - hpdata_t *ps = NULL; - - /* Is eden a perfect fit? */ - if (shard->eden != NULL && shard->eden_len == HUGEPAGE) { - ps = hpa_alloc_ps(tsdn, shard); - if (ps == NULL) { - return NULL; - } - hpdata_init(ps, shard->eden, shard->age_counter++); - shard->eden = NULL; - shard->eden_len = 0; - return ps; - } - - /* - * We're about to try to allocate from eden by splitting. If eden is - * NULL, we have to allocate it too. Otherwise, we just have to - * allocate an edata_t for the new psset. - */ - if (shard->eden == NULL) { - /* Allocate address space, bailing if we fail. */ - void *new_eden = shard->hooks.map(HPA_EDEN_SIZE); - if (new_eden == NULL) { - return NULL; - } - ps = hpa_alloc_ps(tsdn, shard); - if (ps == NULL) { - shard->hooks.unmap(new_eden, HPA_EDEN_SIZE); - return NULL; - } - shard->eden = new_eden; - shard->eden_len = HPA_EDEN_SIZE; - } else { - /* Eden is already nonempty; only need an edata for ps. */ - ps = hpa_alloc_ps(tsdn, shard); - if (ps == NULL) { - return NULL; - } - } - assert(ps != NULL); - assert(shard->eden != NULL); - assert(shard->eden_len > HUGEPAGE); - assert(shard->eden_len % HUGEPAGE == 0); - assert(HUGEPAGE_ADDR2BASE(shard->eden) == shard->eden); - - hpdata_init(ps, shard->eden, shard->age_counter++); - - char *eden_char = (char *)shard->eden; - eden_char += HUGEPAGE; - shard->eden = (void *)eden_char; - shard->eden_len -= HUGEPAGE; - - return ps; -} - /* Returns whether or not we purged anything. */ static bool hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) { @@ -348,7 +398,8 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) { /* Actually do the purging, now that the lock is dropped. */ if (dehugify) { - shard->hooks.dehugify(hpdata_addr_get(to_purge), HUGEPAGE); + shard->central->hooks.dehugify(hpdata_addr_get(to_purge), + HUGEPAGE); } size_t total_purged = 0; uint64_t purges_this_pass = 0; @@ -359,7 +410,7 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) { total_purged += purge_size; assert(total_purged <= HUGEPAGE); purges_this_pass++; - shard->hooks.purge(purge_addr, purge_size); + shard->central->hooks.purge(purge_addr, purge_size); } malloc_mutex_lock(tsdn, &shard->mtx); @@ -406,7 +457,7 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) { /* Make sure that it's been hugifiable for long enough. */ nstime_t time_hugify_allowed = hpdata_time_hugify_allowed(to_hugify); nstime_t nstime; - shard->hooks.curtime(&nstime); + shard->central->hooks.curtime(&nstime); nstime_subtract(&nstime, &time_hugify_allowed); uint64_t millis = nstime_msec(&nstime); if (millis < shard->opts.hugify_delay_ms) { @@ -427,7 +478,7 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) { malloc_mutex_unlock(tsdn, &shard->mtx); - shard->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE); + shard->central->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE); malloc_mutex_lock(tsdn, &shard->mtx); shard->stats.nhugifies++; @@ -604,7 +655,7 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, * deallocations (and allocations of smaller sizes) may still succeed * while we're doing this potentially expensive system call. */ - hpdata_t *ps = hpa_grow(tsdn, shard); + hpdata_t *ps = hpa_central_extract(tsdn, shard->central, size, &oom); if (ps == NULL) { malloc_mutex_unlock(tsdn, &shard->grow_mtx); return nsuccess; @@ -833,7 +884,7 @@ hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) { /* There should be no allocations anywhere. */ assert(hpdata_empty(ps)); psset_remove(&shard->psset, ps); - shard->hooks.unmap(hpdata_addr_get(ps), HUGEPAGE); + shard->central->hooks.unmap(hpdata_addr_get(ps), HUGEPAGE); } } diff --git a/src/jemalloc.c b/src/jemalloc.c index 71efcb61..8d57180e 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1747,7 +1747,19 @@ malloc_init_hard_a0_locked() { if (config_prof) { prof_boot1(); } - arena_boot(&sc_data); + if (opt_hpa && !hpa_supported()) { + malloc_printf(": HPA not supported in the current " + "configuration; %s.", + opt_abort_conf ? "aborting" : "disabling"); + if (opt_abort_conf) { + malloc_abort_invalid_conf(); + } else { + opt_hpa = false; + } + } + if (arena_boot(&sc_data, b0get(), opt_hpa)) { + return true; + } if (tcache_boot(TSDN_NULL, b0get())) { return true; } @@ -1786,7 +1798,7 @@ malloc_init_hard_a0_locked() { hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts; hpa_shard_opts.deferral_allowed = background_thread_enabled(); if (pa_shard_enable_hpa(TSDN_NULL, &a0->pa_shard, - &hpa_hooks_default, &hpa_shard_opts, &opt_hpa_sec_opts)) { + &hpa_shard_opts, &opt_hpa_sec_opts)) { return true; } } diff --git a/src/pa.c b/src/pa.c index 0172dfa7..aebb8e92 100644 --- a/src/pa.c +++ b/src/pa.c @@ -15,10 +15,23 @@ pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) { } bool -pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, - unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx, - nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms, - ssize_t muzzy_decay_ms) { +pa_central_init(pa_central_t *central, base_t *base, bool hpa, + hpa_hooks_t *hpa_hooks) { + bool err; + if (hpa) { + err = hpa_central_init(¢ral->hpa, base, hpa_hooks); + if (err) { + return true; + } + } + return false; +} + +bool +pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central, + emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats, + malloc_mutex_t *stats_mtx, nstime_t *cur_time, size_t oversize_threshold, + ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) { /* This will change eventually, but for now it should hold. */ assert(base_ind_get(base) == ind); if (edata_cache_init(&shard->edata_cache, base)) { @@ -42,6 +55,7 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, shard->stats = stats; memset(shard->stats, 0, sizeof(*shard->stats)); + shard->central = central; shard->emap = emap; shard->base = base; @@ -50,10 +64,9 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, bool pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard, - const hpa_hooks_t *hpa_hooks, const hpa_shard_opts_t *hpa_opts, - const sec_opts_t *hpa_sec_opts) { - if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base, - &shard->edata_cache, shard->ind, hpa_hooks, hpa_opts)) { + const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) { + if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap, + shard->base, &shard->edata_cache, shard->ind, hpa_opts)) { return true; } if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai, diff --git a/test/unit/hpa.c b/test/unit/hpa.c index a9e551fc..2d4fa9b9 100644 --- a/test/unit/hpa.c +++ b/test/unit/hpa.c @@ -13,6 +13,7 @@ struct test_data_s { * test_data_t and the hpa_shard_t; */ hpa_shard_t shard; + hpa_central_t central; base_t *base; edata_cache_t shard_edata_cache; @@ -50,9 +51,12 @@ create_test_data(hpa_hooks_t *hooks, hpa_shard_opts_t *opts) { err = emap_init(&test_data->emap, test_data->base, /* zeroed */ false); assert_false(err, ""); - err = hpa_shard_init(&test_data->shard, &test_data->emap, - test_data->base, &test_data->shard_edata_cache, SHARD_IND, - hooks, opts); + err = hpa_central_init(&test_data->central, test_data->base, hooks); + assert_false(err, ""); + + err = hpa_shard_init(&test_data->shard, &test_data->central, + &test_data->emap, test_data->base, &test_data->shard_edata_cache, + SHARD_IND, opts); assert_false(err, ""); return (hpa_shard_t *)test_data; diff --git a/test/unit/pa.c b/test/unit/pa.c index dacd8e70..4206e85a 100644 --- a/test/unit/pa.c +++ b/test/unit/pa.c @@ -40,6 +40,7 @@ init_test_extent_hooks(extent_hooks_t *hooks) { typedef struct test_data_s test_data_t; struct test_data_s { pa_shard_t shard; + pa_central_t central; base_t *base; emap_t emap; pa_shard_stats_t stats; @@ -63,9 +64,13 @@ test_data_t *init_test_data(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) { nstime_t time; nstime_init(&time, 0); + err = pa_central_init(&test_data->central, base, opt_hpa, + &hpa_hooks_default); + assert_false(err, ""); + const size_t oversize_threshold = 8 * 1024 * 1024; - err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap, - test_data->base, /* ind */ 1, &test_data->stats, + err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->central, + &test_data->emap, test_data->base, /* ind */ 1, &test_data->stats, &test_data->stats_mtx, &time, oversize_threshold, dirty_decay_ms, muzzy_decay_ms); assert_false(err, "");