psset: Add stats

This commit is contained in:
David Goldblatt 2020-09-02 12:59:10 -07:00 committed by David Goldblatt
parent 018b162d67
commit 259c5e3e8f
4 changed files with 178 additions and 13 deletions

View File

@ -218,13 +218,17 @@ struct edata_s {
*/
edata_t *ps;
/*
* If this edata *is* a pageslab, then it has some longest free
* range in it. Track it.
* If this edata *is* a pageslab, then we cache some useful
* information about its associated bitmap.
*/
struct {
/*
* The longest free range a pageslab contains determines
* the heap it lives in. If we know that it didn't
* change after an operation, we can avoid moving it
* between heaps.
*/
uint32_t longest_free_range;
/* Not yet tracked. */
/* uint32_t longest_free_range_pos; */
};
};

View File

@ -21,6 +21,16 @@
*/
#define PSSET_NPSIZES 64
typedef struct psset_bin_stats_s psset_bin_stats_t;
struct psset_bin_stats_s {
/* How many pageslabs are in this bin? */
size_t npageslabs;
/* Of them, how many pages are active? */
size_t nactive;
/* How many are inactive? */
size_t ninactive;
};
typedef struct psset_s psset_t;
struct psset_s {
/*
@ -29,6 +39,12 @@ struct psset_s {
*/
edata_heap_t pageslabs[PSSET_NPSIZES];
bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
/*
* Full slabs don't live in any edata heap. But we still track their
* stats.
*/
psset_bin_stats_t full_slab_stats;
psset_bin_stats_t slab_stats[PSSET_NPSIZES];
};
void psset_init(psset_t *psset);

View File

@ -14,6 +14,48 @@ psset_init(psset_t *psset) {
edata_heap_new(&psset->pageslabs[i]);
}
bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
psset->full_slab_stats.npageslabs = 0;
psset->full_slab_stats.nactive = 0;
psset->full_slab_stats.ninactive = 0;
for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
psset->slab_stats[i].npageslabs = 0;
psset->slab_stats[i].nactive = 0;
psset->slab_stats[i].ninactive = 0;
}
}
/*
* The stats maintenance strategy is simple, but not necessarily obvious.
* edata_nfree and the bitmap must remain consistent at all times. If they
* change while an edata is within an edata_heap (or full), then the associated
* stats bin (or the full bin) must also change. If they change while not in a
* bin (say, in between extraction and reinsertion), then the bin stats need not
* change. If a pageslab is removed from a bin (or becomes nonfull), it should
* no longer contribute to that bin's stats (or the full stats). These help
* ensure we don't miss any heap modification operations.
*/
JEMALLOC_ALWAYS_INLINE void
psset_bin_stats_adjust(psset_bin_stats_t *binstats, edata_t *ps, bool inc) {
size_t mul = inc ? (size_t)1 : (size_t)-1;
size_t npages = edata_size_get(ps) >> LG_PAGE;
size_t ninactive = edata_nfree_get(ps);
size_t nactive = npages - ninactive;
binstats->npageslabs += mul * 1;
binstats->nactive += mul * nactive;
binstats->ninactive += mul * ninactive;
}
static void
psset_edata_heap_remove(psset_t *psset, pszind_t pind, edata_t *ps) {
edata_heap_remove(&psset->pageslabs[pind], ps);
psset_bin_stats_adjust(&psset->slab_stats[pind], ps, /* inc */ false);
}
static void
psset_edata_heap_insert(psset_t *psset, pszind_t pind, edata_t *ps) {
edata_heap_insert(&psset->pageslabs[pind], ps);
psset_bin_stats_adjust(&psset->slab_stats[pind], ps, /* inc */ true);
}
JEMALLOC_ALWAYS_INLINE void
@ -46,7 +88,8 @@ psset_recycle_extract(psset_t *psset, size_t size) {
if (ret == NULL) {
return NULL;
}
edata_heap_remove(&psset->pageslabs[ret_ind], ret);
psset_edata_heap_remove(psset, ret_ind, ret);
if (edata_heap_empty(&psset->pageslabs[ret_ind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info, ret_ind);
}
@ -67,7 +110,7 @@ psset_insert(psset_t *psset, edata_t *ps, size_t largest_range) {
if (edata_heap_empty(&psset->pageslabs[pind])) {
bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
}
edata_heap_insert(&psset->pageslabs[pind], ps);
psset_edata_heap_insert(psset, pind, ps);
}
/*
@ -120,6 +163,9 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
EXTENT_NOT_HEAD);
edata_ps_set(r_edata, ps);
fb_set_range(ps_fb, ps_npages, begin, npages);
edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) - npages));
/* The pageslab isn't in a bin, so no bin stats need to change. */
/*
* OK, we've got to put the pageslab back. First we have to figure out
* where, though; we've only checked run sizes before the pageslab we
@ -144,7 +190,10 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
start = begin + len;
}
edata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range);
if (largest_unchosen_range != 0) {
if (largest_unchosen_range == 0) {
psset_bin_stats_adjust(&psset->full_slab_stats, ps,
/* inc */ true);
} else {
psset_insert(psset, ps, largest_unchosen_range);
}
}
@ -164,8 +213,8 @@ psset_alloc_new(psset_t *psset, edata_t *ps, edata_t *r_edata, size_t size) {
fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
assert(fb_empty(ps_fb, ps_npages));
assert(ps_npages >= (size >> LG_PAGE));
edata_nfree_set(ps, (uint32_t)ps_npages);
psset_ps_alloc_insert(psset, ps, r_edata, size);
}
@ -177,6 +226,11 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
edata_t *ps = edata_ps_get(edata);
fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
size_t ps_old_longest_free_range = edata_longest_free_range_get(ps);
pszind_t old_pind = SC_NPSIZES;
if (ps_old_longest_free_range != 0) {
old_pind = sz_psz2ind(sz_psz_quantize_floor(
ps_old_longest_free_range << LG_PAGE));
}
size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
size_t begin =
@ -184,6 +238,23 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
>> LG_PAGE;
size_t len = edata_size_get(edata) >> LG_PAGE;
fb_unset_range(ps_fb, ps_npages, begin, len);
if (ps_old_longest_free_range == 0) {
/* We were in the (imaginary) full bin; update stats for it. */
psset_bin_stats_adjust(&psset->full_slab_stats, ps,
/* inc */ false);
} else {
/*
* The edata is still in the bin, need to update its
* contribution.
*/
psset->slab_stats[old_pind].nactive -= len;
psset->slab_stats[old_pind].ninactive += len;
}
/*
* Note that we want to do this after the stats updates, since if it was
* full it psset_bin_stats_adjust would have looked at the old version.
*/
edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) + len));
/* We might have just created a new, larger range. */
size_t new_begin = (size_t)(fb_fls(ps_fb, ps_npages, begin) + 1);
@ -215,9 +286,7 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
* and the issue becomes moot).
*/
if (ps_old_longest_free_range > 0) {
pszind_t old_pind = sz_psz2ind(sz_psz_quantize_floor(
ps_old_longest_free_range<< LG_PAGE));
edata_heap_remove(&psset->pageslabs[old_pind], ps);
psset_edata_heap_remove(psset, old_pind, ps);
if (edata_heap_empty(&psset->pageslabs[old_pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info,
(size_t)old_pind);
@ -234,6 +303,6 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
bitmap_unset(psset->bitmap, &psset_bitmap_info,
(size_t)new_pind);
}
edata_heap_insert(&psset->pageslabs[new_pind], ps);
psset_edata_heap_insert(psset, new_pind, ps);
return NULL;
}

View File

@ -295,6 +295,81 @@ TEST_BEGIN(test_multi_pageslab) {
}
TEST_END
static void
stats_expect_empty(psset_bin_stats_t *stats) {
assert_zu_eq(0, stats->npageslabs,
"Supposedly empty bin had positive npageslabs");
expect_zu_eq(0, stats->nactive, "Unexpected nonempty bin"
"Supposedly empty bin had positive nactive");
expect_zu_eq(0, stats->ninactive, "Unexpected nonempty bin"
"Supposedly empty bin had positive ninactive");
}
static void
stats_expect(psset_t *psset, size_t nactive) {
if (nactive == PAGESLAB_PAGES) {
expect_zu_eq(1, psset->full_slab_stats.npageslabs,
"Expected a full slab");
expect_zu_eq(PAGESLAB_PAGES, psset->full_slab_stats.nactive,
"Should have exactly filled the bin");
expect_zu_eq(0, psset->full_slab_stats.ninactive,
"Should never have inactive pages in a full slab");
} else {
stats_expect_empty(&psset->full_slab_stats);
}
size_t ninactive = PAGESLAB_PAGES - nactive;
pszind_t nonempty_pind = PSSET_NPSIZES;
if (ninactive != 0 && ninactive < PAGESLAB_PAGES) {
nonempty_pind = sz_psz2ind(sz_psz_quantize_floor(
ninactive << LG_PAGE));
}
for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
if (i == nonempty_pind) {
assert_zu_eq(1, psset->slab_stats[i].npageslabs,
"Should have found a slab");
expect_zu_eq(nactive, psset->slab_stats[i].nactive,
"Mismatch in active pages");
expect_zu_eq(ninactive, psset->slab_stats[i].ninactive,
"Mismatch in inactive pages");
} else {
stats_expect_empty(&psset->slab_stats[i]);
}
}
}
TEST_BEGIN(test_stats) {
bool err;
edata_t pageslab;
memset(&pageslab, 0, sizeof(pageslab));
edata_t alloc[PAGESLAB_PAGES];
edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
/* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
/* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
EXTENT_IS_HEAD);
psset_t psset;
psset_init(&psset);
stats_expect(&psset, 0);
edata_init_test(&alloc[0]);
psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
stats_expect(&psset, i);
edata_init_test(&alloc[i]);
err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
expect_false(err, "Nonempty psset failed page allocation.");
}
stats_expect(&psset, PAGESLAB_PAGES);
for (ssize_t i = PAGESLAB_PAGES - 1; i >= 0; i--) {
edata_t *ps = psset_dalloc(&psset, &alloc[i]);
expect_true((ps == NULL) == (i != 0),
"psset_dalloc should only evict a slab on the last free");
stats_expect(&psset, i);
}
}
TEST_END
int
main(void) {
return test_no_reentrancy(
@ -302,5 +377,6 @@ main(void) {
test_fill,
test_reuse,
test_evict,
test_multi_pageslab);
test_multi_pageslab,
test_stats);
}