diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index 7027cff7..d8189666 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -26,25 +26,28 @@ typedef struct psset_bin_stats_s psset_bin_stats_t; struct psset_bin_stats_s { /* How many pageslabs are in this bin? */ - size_t npageslabs_huge; - size_t npageslabs_nonhuge; + size_t npageslabs; /* Of them, how many pages are active? */ - size_t nactive_huge; - size_t nactive_nonhuge; + size_t nactive; /* How many are inactive? */ - size_t ninactive_huge; - size_t ninactive_nonhuge; + size_t ninactive; }; -/* Used only by CTL; not actually stored here (i.e., all derived). */ typedef struct psset_stats_s psset_stats_t; struct psset_stats_s { + + /* + * The second index is huge stats; nonfull_slabs[pszind][0] contains + * stats for the non-huge slabs in bucket pszind, while + * nonfull_slabs[pszind][1] contains stats for the huge slabs. + */ + psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2]; + /* * Full slabs don't live in any edata heap. But we still track their * stats. */ - psset_bin_stats_t full_slabs; - psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES]; + psset_bin_stats_t full_slabs[2]; }; typedef struct psset_s psset_t; diff --git a/src/ctl.c b/src/ctl.c index 8871fd15..516add4e 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -3525,46 +3525,47 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies, arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t); -/* Full, huge */ -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.npageslabs_huge, - size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.nactive_huge, size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.ninactive_huge, size_t); - /* Full, nonhuge */ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.npageslabs_nonhuge, + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.nactive_nonhuge, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.ninactive_nonhuge, size_t); + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ninactive, size_t); -/* Nonfull, huge */ -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].npageslabs_huge, - size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].nactive_huge, - size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_huge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].ninactive_huge, +/* Full, huge */ +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ninactive, size_t); /* Nonfull, nonhuge */ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].npageslabs_nonhuge, + arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].npageslabs, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].nactive_nonhuge, + arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].nactive, size_t); CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_nonhuge, - arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].ninactive_nonhuge, + arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].ninactive, size_t); +/* Nonfull, huge */ +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].npageslabs, + size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].nactive, + size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_huge, + arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].ninactive, + size_t); + + static const ctl_named_node_t * stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) { diff --git a/src/hpa.c b/src/hpa.c index 4069c1ea..a206cffe 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -662,12 +662,9 @@ hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) { static void hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) { - assert(bin_stats->npageslabs_huge == 0); - assert(bin_stats->nactive_huge == 0); - assert(bin_stats->ninactive_huge == 0); - assert(bin_stats->npageslabs_nonhuge == 0); - assert(bin_stats->nactive_nonhuge == 0); - assert(bin_stats->ninactive_nonhuge == 0); + assert(bin_stats->npageslabs == 0); + assert(bin_stats->nactive == 0); + assert(bin_stats->ninactive == 0); } static void @@ -675,10 +672,12 @@ hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) { malloc_mutex_assert_owner(tsdn, &shard->mtx); hpdata_t *ps = psset_fit(psset, PAGE); assert(ps == NULL); - hpa_shard_assert_stats_empty(&psset->stats.full_slabs); - for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { - hpa_shard_assert_stats_empty( - &psset->stats.nonfull_slabs[i]); + for (int huge = 0; huge <= 1; huge++) { + hpa_shard_assert_stats_empty(&psset->stats.full_slabs[huge]); + for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { + hpa_shard_assert_stats_empty( + &psset->stats.nonfull_slabs[i][huge]); + } } } diff --git a/src/psset.c b/src/psset.c index 688cd620..a91653f4 100644 --- a/src/psset.c +++ b/src/psset.c @@ -19,21 +19,20 @@ psset_init(psset_t *psset) { static void psset_bin_stats_accum(psset_bin_stats_t *dst, psset_bin_stats_t *src) { - dst->npageslabs_huge += src->npageslabs_huge; - dst->nactive_huge += src->nactive_huge; - dst->ninactive_huge += src->ninactive_huge; - - dst->npageslabs_nonhuge += src->npageslabs_nonhuge; - dst->nactive_nonhuge += src->nactive_nonhuge; - dst->ninactive_nonhuge += src->ninactive_nonhuge; + dst->npageslabs += src->npageslabs; + dst->nactive += src->nactive; + dst->ninactive += src->ninactive; } void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) { - psset_bin_stats_accum(&dst->full_slabs, &src->full_slabs); + psset_bin_stats_accum(&dst->full_slabs[0], &src->full_slabs[0]); + psset_bin_stats_accum(&dst->full_slabs[1], &src->full_slabs[1]); for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { - psset_bin_stats_accum(&dst->nonfull_slabs[i], - &src->nonfull_slabs[i]); + psset_bin_stats_accum(&dst->nonfull_slabs[i][0], + &src->nonfull_slabs[i][0]); + psset_bin_stats_accum(&dst->nonfull_slabs[i][1], + &src->nonfull_slabs[i][1]); } } @@ -50,42 +49,34 @@ psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) { JEMALLOC_ALWAYS_INLINE void psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps, bool insert) { - size_t *npageslabs_dst = hpdata_huge_get(ps) - ? &binstats->npageslabs_huge : &binstats->npageslabs_nonhuge; - size_t *nactive_dst = hpdata_huge_get(ps) - ? &binstats->nactive_huge : &binstats->nactive_nonhuge; - size_t *ninactive_dst = hpdata_huge_get(ps) - ? &binstats->ninactive_huge : &binstats->ninactive_nonhuge; - - size_t nactive = hpdata_nactive_get(ps); - size_t ninactive = HUGEPAGE_PAGES - nactive; - size_t mul = insert ? (size_t)1 : (size_t)-1; - *npageslabs_dst += mul * 1; - *nactive_dst += mul * nactive; - *ninactive_dst += mul * ninactive; + size_t huge_idx = (size_t)hpdata_huge_get(ps); + binstats[huge_idx].npageslabs += mul * 1; + size_t nactive = hpdata_nactive_get(ps); + binstats[huge_idx].nactive += mul * nactive; + binstats[huge_idx].ninactive += mul * (HUGEPAGE_PAGES - nactive); } static void psset_bin_stats_insert(psset_bin_stats_t *binstats, hpdata_t *ps) { - psset_bin_stats_insert_remove(binstats, ps, /* insert */ true); + psset_bin_stats_insert_remove(binstats, ps, true); } static void psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) { - psset_bin_stats_insert_remove(binstats, ps, /* insert */ false); + psset_bin_stats_insert_remove(binstats, ps, false); } static void psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) { hpdata_age_heap_remove(&psset->pageslabs[pind], ps); - psset_bin_stats_remove(&psset->stats.nonfull_slabs[pind], ps); + psset_bin_stats_remove(psset->stats.nonfull_slabs[pind], ps); } static void psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) { hpdata_age_heap_insert(&psset->pageslabs[pind], ps); - psset_bin_stats_insert(&psset->stats.nonfull_slabs[pind], ps); + psset_bin_stats_insert(psset->stats.nonfull_slabs[pind], ps); } void @@ -101,7 +92,7 @@ psset_insert(psset_t *psset, hpdata_t *ps) { * We don't ned to track full slabs; just pretend to for stats * purposes. See the comment at psset_bin_stats_adjust. */ - psset_bin_stats_insert(&psset->stats.full_slabs, ps); + psset_bin_stats_insert(psset->stats.full_slabs, ps); return; } @@ -124,7 +115,7 @@ psset_remove(psset_t *psset, hpdata_t *ps) { size_t longest_free_range = hpdata_longest_free_range_get(ps); if (longest_free_range == 0) { - psset_bin_stats_remove(&psset->stats.full_slabs, ps); + psset_bin_stats_remove(psset->stats.full_slabs, ps); return; } diff --git a/test/unit/psset.c b/test/unit/psset.c index 6f35fa8d..020a8325 100644 --- a/test/unit/psset.c +++ b/test/unit/psset.c @@ -321,26 +321,26 @@ TEST_END static void stats_expect_empty(psset_bin_stats_t *stats) { - assert_zu_eq(0, stats->npageslabs_nonhuge, + assert_zu_eq(0, stats->npageslabs, "Supposedly empty bin had positive npageslabs"); - expect_zu_eq(0, stats->nactive_nonhuge, "Unexpected nonempty bin" + expect_zu_eq(0, stats->nactive, "Unexpected nonempty bin" "Supposedly empty bin had positive nactive"); - expect_zu_eq(0, stats->ninactive_nonhuge, "Unexpected nonempty bin" + expect_zu_eq(0, stats->ninactive, "Unexpected nonempty bin" "Supposedly empty bin had positive ninactive"); } static void stats_expect(psset_t *psset, size_t nactive) { if (nactive == HUGEPAGE_PAGES) { - expect_zu_eq(1, psset->stats.full_slabs.npageslabs_nonhuge, + expect_zu_eq(1, psset->stats.full_slabs[0].npageslabs, "Expected a full slab"); expect_zu_eq(HUGEPAGE_PAGES, - psset->stats.full_slabs.nactive_nonhuge, + psset->stats.full_slabs[0].nactive, "Should have exactly filled the bin"); - expect_zu_eq(0, psset->stats.full_slabs.ninactive_nonhuge, + expect_zu_eq(0, psset->stats.full_slabs[0].ninactive, "Should never have inactive pages in a full slab"); } else { - stats_expect_empty(&psset->stats.full_slabs); + stats_expect_empty(&psset->stats.full_slabs[0]); } size_t ninactive = HUGEPAGE_PAGES - nactive; pszind_t nonempty_pind = PSSET_NPSIZES; @@ -351,16 +351,16 @@ stats_expect(psset_t *psset, size_t nactive) { for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { if (i == nonempty_pind) { assert_zu_eq(1, - psset->stats.nonfull_slabs[i].npageslabs_nonhuge, + psset->stats.nonfull_slabs[i][0].npageslabs, "Should have found a slab"); expect_zu_eq(nactive, - psset->stats.nonfull_slabs[i].nactive_nonhuge, + psset->stats.nonfull_slabs[i][0].nactive, "Mismatch in active pages"); expect_zu_eq(ninactive, - psset->stats.nonfull_slabs[i].ninactive_nonhuge, + psset->stats.nonfull_slabs[i][0].ninactive, "Mismatch in inactive pages"); } else { - stats_expect_empty(&psset->stats.nonfull_slabs[i]); + stats_expect_empty(&psset->stats.nonfull_slabs[i][0]); } } }