project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

psset stats: Simplify handling.

Browse Source 
We can treat the huge and nonhuge cases uniformly using huge state as an array
index.
This commit is contained in:
David Goldblatt 2020-12-03 18:32:42 -08:00 committed by David Goldblatt
parent 94cd9444c5
commit 55e0f60ca1
5 changed files with 77 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/jemalloc/internal/psset.h
View File

@ -26,25 +26,28 @@
typedef struct psset_bin_stats_s psset_bin_stats_t;
struct psset_bin_stats_s {
/* How many pageslabs are in this bin? */
size_t npageslabs_huge;
size_t npageslabs_nonhuge;
size_t npageslabs;
/* Of them, how many pages are active? */
size_t nactive_huge;
size_t nactive_nonhuge;
size_t nactive;
/* How many are inactive? */
size_t ninactive_huge;
size_t ninactive_nonhuge;
size_t ninactive;
};
/* Used only by CTL; not actually stored here (i.e., all derived). */
typedef struct psset_stats_s psset_stats_t;
struct psset_stats_s {
/*
* The second index is huge stats; nonfull_slabs[pszind][0] contains
* stats for the non-huge slabs in bucket pszind, while
* nonfull_slabs[pszind][1] contains stats for the huge slabs.
*/
psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2];
/*
* Full slabs don't live in any edata heap. But we still track their
* stats.
*/
psset_bin_stats_t full_slabs;
psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES];
psset_bin_stats_t full_slabs[2];
};
typedef struct psset_s psset_t;

49
src/ctl.c
View File

@ -3525,46 +3525,47 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies,
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies,
arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t);
/* Full, huge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.npageslabs_huge,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.nactive_huge, size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.ninactive_huge, size_t);
/* Full, nonhuge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.npageslabs_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.nactive_nonhuge, size_t);
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive, size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs.ninactive_nonhuge, size_t);
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ninactive, size_t);
/* Nonfull, huge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].npageslabs_huge,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].nactive_huge,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].ninactive_huge,
/* Full, huge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive, size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ninactive, size_t);
/* Nonfull, nonhuge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].npageslabs_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].npageslabs,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].nactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].nactive,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]].ninactive_nonhuge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0].ninactive,
size_t);
/* Nonfull, huge */
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].npageslabs,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].nactive,
size_t);
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive_huge,
arenas_i(mib[2])->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1].ninactive,
size_t);
static const ctl_named_node_t *
stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib,
size_t miblen, size_t j) {

15
src/hpa.c
View File

@ -662,12 +662,9 @@ hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) {
static void
hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) {
assert(bin_stats->npageslabs_huge == 0);
assert(bin_stats->nactive_huge == 0);
assert(bin_stats->ninactive_huge == 0);
assert(bin_stats->npageslabs_nonhuge == 0);
assert(bin_stats->nactive_nonhuge == 0);
assert(bin_stats->ninactive_nonhuge == 0);
assert(bin_stats->npageslabs == 0);
assert(bin_stats->nactive == 0);
assert(bin_stats->ninactive == 0);
}
static void
@ -675,10 +672,12 @@ hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
hpdata_t *ps = psset_fit(psset, PAGE);
assert(ps == NULL);
hpa_shard_assert_stats_empty(&psset->stats.full_slabs);
for (int huge = 0; huge <= 1; huge++) {
hpa_shard_assert_stats_empty(&psset->stats.full_slabs[huge]);
for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
hpa_shard_assert_stats_empty(
&psset->stats.nonfull_slabs[i]);
&psset->stats.nonfull_slabs[i][huge]);
}
}
}

49
src/psset.c
View File

@ -19,21 +19,20 @@ psset_init(psset_t *psset) {
static void
psset_bin_stats_accum(psset_bin_stats_t *dst, psset_bin_stats_t *src) {
dst->npageslabs_huge += src->npageslabs_huge;
dst->nactive_huge += src->nactive_huge;
dst->ninactive_huge += src->ninactive_huge;
dst->npageslabs_nonhuge += src->npageslabs_nonhuge;
dst->nactive_nonhuge += src->nactive_nonhuge;
dst->ninactive_nonhuge += src->ninactive_nonhuge;
dst->npageslabs += src->npageslabs;
dst->nactive += src->nactive;
dst->ninactive += src->ninactive;
}
void
psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) {
psset_bin_stats_accum(&dst->full_slabs, &src->full_slabs);
psset_bin_stats_accum(&dst->full_slabs[0], &src->full_slabs[0]);
psset_bin_stats_accum(&dst->full_slabs[1], &src->full_slabs[1]);
for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
psset_bin_stats_accum(&dst->nonfull_slabs[i],
&src->nonfull_slabs[i]);
psset_bin_stats_accum(&dst->nonfull_slabs[i][0],
&src->nonfull_slabs[i][0]);
psset_bin_stats_accum(&dst->nonfull_slabs[i][1],
&src->nonfull_slabs[i][1]);
}
}
@ -50,42 +49,34 @@ psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) {
JEMALLOC_ALWAYS_INLINE void
psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps,
bool insert) {
size_t *npageslabs_dst = hpdata_huge_get(ps)
? &binstats->npageslabs_huge : &binstats->npageslabs_nonhuge;
size_t *nactive_dst = hpdata_huge_get(ps)
? &binstats->nactive_huge : &binstats->nactive_nonhuge;
size_t *ninactive_dst = hpdata_huge_get(ps)
? &binstats->ninactive_huge : &binstats->ninactive_nonhuge;
size_t nactive = hpdata_nactive_get(ps);
size_t ninactive = HUGEPAGE_PAGES - nactive;
size_t mul = insert ? (size_t)1 : (size_t)-1;
*npageslabs_dst += mul * 1;
*nactive_dst += mul * nactive;
*ninactive_dst += mul * ninactive;
size_t huge_idx = (size_t)hpdata_huge_get(ps);
binstats[huge_idx].npageslabs += mul * 1;
size_t nactive = hpdata_nactive_get(ps);
binstats[huge_idx].nactive += mul * nactive;
binstats[huge_idx].ninactive += mul * (HUGEPAGE_PAGES - nactive);
}
static void
psset_bin_stats_insert(psset_bin_stats_t *binstats, hpdata_t *ps) {
psset_bin_stats_insert_remove(binstats, ps, /* insert */ true);
psset_bin_stats_insert_remove(binstats, ps, true);
}
static void
psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) {
psset_bin_stats_insert_remove(binstats, ps, /* insert */ false);
psset_bin_stats_insert_remove(binstats, ps, false);
}
static void
psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
psset_bin_stats_remove(&psset->stats.nonfull_slabs[pind], ps);
psset_bin_stats_remove(psset->stats.nonfull_slabs[pind], ps);
}
static void
psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
psset_bin_stats_insert(&psset->stats.nonfull_slabs[pind], ps);
psset_bin_stats_insert(psset->stats.nonfull_slabs[pind], ps);
}
void
@ -101,7 +92,7 @@ psset_insert(psset_t *psset, hpdata_t *ps) {
* We don't ned to track full slabs; just pretend to for stats
* purposes. See the comment at psset_bin_stats_adjust.
*/
psset_bin_stats_insert(&psset->stats.full_slabs, ps);
psset_bin_stats_insert(psset->stats.full_slabs, ps);
return;
}
@ -124,7 +115,7 @@ psset_remove(psset_t *psset, hpdata_t *ps) {
size_t longest_free_range = hpdata_longest_free_range_get(ps);
if (longest_free_range == 0) {
psset_bin_stats_remove(&psset->stats.full_slabs, ps);
psset_bin_stats_remove(psset->stats.full_slabs, ps);
return;
}

22
test/unit/psset.c
View File

@ -321,26 +321,26 @@ TEST_END
static void
stats_expect_empty(psset_bin_stats_t *stats) {
assert_zu_eq(0, stats->npageslabs_nonhuge,
assert_zu_eq(0, stats->npageslabs,
"Supposedly empty bin had positive npageslabs");
expect_zu_eq(0, stats->nactive_nonhuge, "Unexpected nonempty bin"
expect_zu_eq(0, stats->nactive, "Unexpected nonempty bin"
"Supposedly empty bin had positive nactive");
expect_zu_eq(0, stats->ninactive_nonhuge, "Unexpected nonempty bin"
expect_zu_eq(0, stats->ninactive, "Unexpected nonempty bin"
"Supposedly empty bin had positive ninactive");
}
static void
stats_expect(psset_t *psset, size_t nactive) {
if (nactive == HUGEPAGE_PAGES) {
expect_zu_eq(1, psset->stats.full_slabs.npageslabs_nonhuge,
expect_zu_eq(1, psset->stats.full_slabs[0].npageslabs,
"Expected a full slab");
expect_zu_eq(HUGEPAGE_PAGES,
psset->stats.full_slabs.nactive_nonhuge,
psset->stats.full_slabs[0].nactive,
"Should have exactly filled the bin");
expect_zu_eq(0, psset->stats.full_slabs.ninactive_nonhuge,
expect_zu_eq(0, psset->stats.full_slabs[0].ninactive,
"Should never have inactive pages in a full slab");
} else {
stats_expect_empty(&psset->stats.full_slabs);
stats_expect_empty(&psset->stats.full_slabs[0]);
}
size_t ninactive = HUGEPAGE_PAGES - nactive;
pszind_t nonempty_pind = PSSET_NPSIZES;
@ -351,16 +351,16 @@ stats_expect(psset_t *psset, size_t nactive) {
for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
if (i == nonempty_pind) {
assert_zu_eq(1,
psset->stats.nonfull_slabs[i].npageslabs_nonhuge,
psset->stats.nonfull_slabs[i][0].npageslabs,
"Should have found a slab");
expect_zu_eq(nactive,
psset->stats.nonfull_slabs[i].nactive_nonhuge,
psset->stats.nonfull_slabs[i][0].nactive,
"Mismatch in active pages");
expect_zu_eq(ninactive,
psset->stats.nonfull_slabs[i].ninactive_nonhuge,
psset->stats.nonfull_slabs[i][0].ninactive,
"Mismatch in inactive pages");
} else {
stats_expect_empty(&psset->stats.nonfull_slabs[i]);
stats_expect_empty(&psset->stats.nonfull_slabs[i][0]);
}
}
}