project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

psset stats: Simplify handling.

Browse Source 
We can treat the huge and nonhuge cases uniformly using huge state as an array
index.
This commit is contained in:
David Goldblatt 2020-12-03 18:32:42 -08:00 committed by David Goldblatt
parent 94cd9444c5
commit 55e0f60ca1
5 changed files with 77 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/jemalloc/internal/psset.h
View File

@ -26,25 +26,28 @@
typedef struct psset_bin_stats_s psset_bin_stats_t; typedef struct psset_bin_stats_s psset_bin_stats_t;
struct psset_bin_stats_s { struct psset_bin_stats_s {
/* How many pageslabs are in this bin? */ /* How many pageslabs are in this bin? */
size_t npageslabs_huge; size_t npageslabs;
size_t npageslabs_nonhuge;
/* Of them, how many pages are active? */ /* Of them, how many pages are active? */
size_t nactive_huge; size_t nactive;
size_t nactive_nonhuge;
/* How many are inactive? */ /* How many are inactive? */
size_t ninactive_huge; size_t ninactive;
size_t ninactive_nonhuge;
}; };
/* Used only by CTL; not actually stored here (i.e., all derived). */
typedef struct psset_stats_s psset_stats_t; typedef struct psset_stats_s psset_stats_t;
struct psset_stats_s { struct psset_stats_s {
/*
* The second index is huge stats; nonfull_slabs[pszind][0] contains
* stats for the non-huge slabs in bucket pszind, while
* nonfull_slabs[pszind][1] contains stats for the huge slabs.
*/
psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2];
/* /*
* Full slabs don't live in any edata heap. But we still track their * Full slabs don't live in any edata heap. But we still track their
* stats. * stats.
*/ */
psset_bin_stats_t full_slabs; psset_bin_stats_t full_slabs[2];
psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES];
}; };
typedef struct psset_s psset_t; typedef struct psset_s psset_t;

49
src/ctl.c
View File

@ -3525,46 +3525,47 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies,
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies,
arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t); arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t);
/* Full, huge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.npageslabs_huge,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.nactive_huge, size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.ninactive_huge, size_t);
/* Full, nonhuge */ /* Full, nonhuge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.npageslabs_nonhuge, arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs,
size_t); size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.nactive_nonhuge, size_t); arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive, size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_nonhuge, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.ninactive_nonhuge, size_t); arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ninactive, size_t);
/* Nonfull, huge */ /* Full, huge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].npageslabs_huge, arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].nactive_huge,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].ninactive_huge,
size_t); size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive, size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ninactive, size_t);
/* Nonfull, nonhuge */ /* Nonfull, nonhuge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].npageslabs_nonhuge, arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].npageslabs,
size_t); size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].nactive_nonhuge, arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].nactive,
size_t); size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_nonhuge, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].ninactive_nonhuge, arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].ninactive,
size_t); size_t);
/* Nonfull, huge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].npageslabs,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].nactive,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].ninactive,
size_t);
static const ctl_named_node_t * static const ctl_named_node_t *
stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib, stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib,
size_t miblen, size_t j) { size_t miblen, size_t j) {

19
src/hpa.c
View File

@ -662,12 +662,9 @@ hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) {
static void static void
hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) { hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) {
assert(bin_stats->npageslabs_huge == 0); assert(bin_stats->npageslabs == 0);
assert(bin_stats->nactive_huge == 0); assert(bin_stats->nactive == 0);
assert(bin_stats->ninactive_huge == 0); assert(bin_stats->ninactive == 0);
assert(bin_stats->npageslabs_nonhuge == 0);
assert(bin_stats->nactive_nonhuge == 0);
assert(bin_stats->ninactive_nonhuge == 0);
} }
static void static void
@ -675,10 +672,12 @@ hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) {
malloc_mutex_assert_owner(tsdn, &shard->mtx); malloc_mutex_assert_owner(tsdn, &shard->mtx);
hpdata_t *ps = psset_fit(psset, PAGE); hpdata_t *ps = psset_fit(psset, PAGE);
assert(ps == NULL); assert(ps == NULL);
hpa_shard_assert_stats_empty(&psset->stats.full_slabs); for (int huge = 0; huge <= 1; huge++) {
for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { hpa_shard_assert_stats_empty(&psset->stats.full_slabs[huge]);
hpa_shard_assert_stats_empty( for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
&psset->stats.nonfull_slabs[i]); hpa_shard_assert_stats_empty(
&psset->stats.nonfull_slabs[i][huge]);
}
} }
} }

49
src/psset.c
View File

@ -19,21 +19,20 @@ psset_init(psset_t *psset) {
static void static void
psset_bin_stats_accum(psset_bin_stats_t *dst, psset_bin_stats_t *src) { psset_bin_stats_accum(psset_bin_stats_t *dst, psset_bin_stats_t *src) {
dst->npageslabs_huge += src->npageslabs_huge; dst->npageslabs += src->npageslabs;
dst->nactive_huge += src->nactive_huge; dst->nactive += src->nactive;
dst->ninactive_huge += src->ninactive_huge; dst->ninactive += src->ninactive;
dst->npageslabs_nonhuge += src->npageslabs_nonhuge;
dst->nactive_nonhuge += src->nactive_nonhuge;
dst->ninactive_nonhuge += src->ninactive_nonhuge;
} }
void void
psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) { psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) {
psset_bin_stats_accum(&dst->full_slabs, &src->full_slabs); psset_bin_stats_accum(&dst->full_slabs[0], &src->full_slabs[0]);
psset_bin_stats_accum(&dst->full_slabs[1], &src->full_slabs[1]);
for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
psset_bin_stats_accum(&dst->nonfull_slabs[i], psset_bin_stats_accum(&dst->nonfull_slabs[i][0],
&src->nonfull_slabs[i]); &src->nonfull_slabs[i][0]);
psset_bin_stats_accum(&dst->nonfull_slabs[i][1],
&src->nonfull_slabs[i][1]);
} }
} }
@ -50,42 +49,34 @@ psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) {
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps, psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps,
bool insert) { bool insert) {
size_t *npageslabs_dst = hpdata_huge_get(ps)
? &binstats->npageslabs_huge : &binstats->npageslabs_nonhuge;
size_t *nactive_dst = hpdata_huge_get(ps)
? &binstats->nactive_huge : &binstats->nactive_nonhuge;
size_t *ninactive_dst = hpdata_huge_get(ps)
? &binstats->ninactive_huge : &binstats->ninactive_nonhuge;
size_t nactive = hpdata_nactive_get(ps);
size_t ninactive = HUGEPAGE_PAGES - nactive;
size_t mul = insert ? (size_t)1 : (size_t)-1; size_t mul = insert ? (size_t)1 : (size_t)-1;
*npageslabs_dst += mul * 1; size_t huge_idx = (size_t)hpdata_huge_get(ps);
*nactive_dst += mul * nactive; binstats[huge_idx].npageslabs += mul * 1;
*ninactive_dst += mul * ninactive; size_t nactive = hpdata_nactive_get(ps);
binstats[huge_idx].nactive += mul * nactive;
binstats[huge_idx].ninactive += mul * (HUGEPAGE_PAGES - nactive);
} }
static void static void
psset_bin_stats_insert(psset_bin_stats_t *binstats, hpdata_t *ps) { psset_bin_stats_insert(psset_bin_stats_t *binstats, hpdata_t *ps) {
psset_bin_stats_insert_remove(binstats, ps, /* insert */ true); psset_bin_stats_insert_remove(binstats, ps, true);
} }
static void static void
psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) { psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) {
psset_bin_stats_insert_remove(binstats, ps, /* insert */ false); psset_bin_stats_insert_remove(binstats, ps, false);
} }
static void static void
psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) { psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
hpdata_age_heap_remove(&psset->pageslabs[pind], ps); hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
psset_bin_stats_remove(&psset->stats.nonfull_slabs[pind], ps); psset_bin_stats_remove(psset->stats.nonfull_slabs[pind], ps);
} }
static void static void
psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) { psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
hpdata_age_heap_insert(&psset->pageslabs[pind], ps); hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
psset_bin_stats_insert(&psset->stats.nonfull_slabs[pind], ps); psset_bin_stats_insert(psset->stats.nonfull_slabs[pind], ps);
} }
void void
@ -101,7 +92,7 @@ psset_insert(psset_t *psset, hpdata_t *ps) {
* We don't ned to track full slabs; just pretend to for stats * We don't ned to track full slabs; just pretend to for stats
* purposes. See the comment at psset_bin_stats_adjust. * purposes. See the comment at psset_bin_stats_adjust.
*/ */
psset_bin_stats_insert(&psset->stats.full_slabs, ps); psset_bin_stats_insert(psset->stats.full_slabs, ps);
return; return;
} }
@ -124,7 +115,7 @@ psset_remove(psset_t *psset, hpdata_t *ps) {
size_t longest_free_range = hpdata_longest_free_range_get(ps); size_t longest_free_range = hpdata_longest_free_range_get(ps);
if (longest_free_range == 0) { if (longest_free_range == 0) {
psset_bin_stats_remove(&psset->stats.full_slabs, ps); psset_bin_stats_remove(psset->stats.full_slabs, ps);
return; return;
} }

22
test/unit/psset.c
View File

@ -321,26 +321,26 @@ TEST_END
static void static void
stats_expect_empty(psset_bin_stats_t *stats) { stats_expect_empty(psset_bin_stats_t *stats) {
assert_zu_eq(0, stats->npageslabs_nonhuge, assert_zu_eq(0, stats->npageslabs,
"Supposedly empty bin had positive npageslabs"); "Supposedly empty bin had positive npageslabs");
expect_zu_eq(0, stats->nactive_nonhuge, "Unexpected nonempty bin" expect_zu_eq(0, stats->nactive, "Unexpected nonempty bin"
"Supposedly empty bin had positive nactive"); "Supposedly empty bin had positive nactive");
expect_zu_eq(0, stats->ninactive_nonhuge, "Unexpected nonempty bin" expect_zu_eq(0, stats->ninactive, "Unexpected nonempty bin"
"Supposedly empty bin had positive ninactive"); "Supposedly empty bin had positive ninactive");
} }
static void static void
stats_expect(psset_t *psset, size_t nactive) { stats_expect(psset_t *psset, size_t nactive) {
if (nactive == HUGEPAGE_PAGES) { if (nactive == HUGEPAGE_PAGES) {
expect_zu_eq(1, psset->stats.full_slabs.npageslabs_nonhuge, expect_zu_eq(1, psset->stats.full_slabs[0].npageslabs,
"Expected a full slab"); "Expected a full slab");
expect_zu_eq(HUGEPAGE_PAGES, expect_zu_eq(HUGEPAGE_PAGES,
psset->stats.full_slabs.nactive_nonhuge, psset->stats.full_slabs[0].nactive,
"Should have exactly filled the bin"); "Should have exactly filled the bin");
expect_zu_eq(0, psset->stats.full_slabs.ninactive_nonhuge, expect_zu_eq(0, psset->stats.full_slabs[0].ninactive,
"Should never have inactive pages in a full slab"); "Should never have inactive pages in a full slab");
} else { } else {
stats_expect_empty(&psset->stats.full_slabs); stats_expect_empty(&psset->stats.full_slabs[0]);
} }
size_t ninactive = HUGEPAGE_PAGES - nactive; size_t ninactive = HUGEPAGE_PAGES - nactive;
pszind_t nonempty_pind = PSSET_NPSIZES; pszind_t nonempty_pind = PSSET_NPSIZES;
@ -351,16 +351,16 @@ stats_expect(psset_t *psset, size_t nactive) {
for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
if (i == nonempty_pind) { if (i == nonempty_pind) {
assert_zu_eq(1, assert_zu_eq(1,
psset->stats.nonfull_slabs[i].npageslabs_nonhuge, psset->stats.nonfull_slabs[i][0].npageslabs,
"Should have found a slab"); "Should have found a slab");
expect_zu_eq(nactive, expect_zu_eq(nactive,
psset->stats.nonfull_slabs[i].nactive_nonhuge, psset->stats.nonfull_slabs[i][0].nactive,
"Mismatch in active pages"); "Mismatch in active pages");
expect_zu_eq(ninactive, expect_zu_eq(ninactive,
psset->stats.nonfull_slabs[i].ninactive_nonhuge, psset->stats.nonfull_slabs[i][0].ninactive,
"Mismatch in inactive pages"); "Mismatch in inactive pages");
} else { } else {
stats_expect_empty(&psset->stats.nonfull_slabs[i]); stats_expect_empty(&psset->stats.nonfull_slabs[i][0]);
} }
} }
} }