diff --git a/include/jemalloc/internal/edata.h b/include/jemalloc/internal/edata.h index 4fee76bf..f175af94 100644 --- a/include/jemalloc/internal/edata.h +++ b/include/jemalloc/internal/edata.h @@ -218,13 +218,17 @@ struct edata_s { */ edata_t *ps; /* - * If this edata *is* a pageslab, then it has some longest free - * range in it. Track it. + * If this edata *is* a pageslab, then we cache some useful + * information about its associated bitmap. */ struct { + /* + * The longest free range a pageslab contains determines + * the heap it lives in. If we know that it didn't + * change after an operation, we can avoid moving it + * between heaps. + */ uint32_t longest_free_range; - /* Not yet tracked. */ - /* uint32_t longest_free_range_pos; */ }; }; diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index 8f3f9ee7..abbfc241 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -21,6 +21,16 @@ */ #define PSSET_NPSIZES 64 +typedef struct psset_bin_stats_s psset_bin_stats_t; +struct psset_bin_stats_s { + /* How many pageslabs are in this bin? */ + size_t npageslabs; + /* Of them, how many pages are active? */ + size_t nactive; + /* How many are inactive? */ + size_t ninactive; +}; + typedef struct psset_s psset_t; struct psset_s { /* @@ -29,6 +39,12 @@ struct psset_s { */ edata_heap_t pageslabs[PSSET_NPSIZES]; bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)]; + /* + * Full slabs don't live in any edata heap. But we still track their + * stats. + */ + psset_bin_stats_t full_slab_stats; + psset_bin_stats_t slab_stats[PSSET_NPSIZES]; }; void psset_init(psset_t *psset); diff --git a/src/psset.c b/src/psset.c index 9675a0d1..04d3548f 100644 --- a/src/psset.c +++ b/src/psset.c @@ -14,6 +14,48 @@ psset_init(psset_t *psset) { edata_heap_new(&psset->pageslabs[i]); } bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true); + psset->full_slab_stats.npageslabs = 0; + psset->full_slab_stats.nactive = 0; + psset->full_slab_stats.ninactive = 0; + for (unsigned i = 0; i < PSSET_NPSIZES; i++) { + psset->slab_stats[i].npageslabs = 0; + psset->slab_stats[i].nactive = 0; + psset->slab_stats[i].ninactive = 0; + } +} + +/* + * The stats maintenance strategy is simple, but not necessarily obvious. + * edata_nfree and the bitmap must remain consistent at all times. If they + * change while an edata is within an edata_heap (or full), then the associated + * stats bin (or the full bin) must also change. If they change while not in a + * bin (say, in between extraction and reinsertion), then the bin stats need not + * change. If a pageslab is removed from a bin (or becomes nonfull), it should + * no longer contribute to that bin's stats (or the full stats). These help + * ensure we don't miss any heap modification operations. + */ +JEMALLOC_ALWAYS_INLINE void +psset_bin_stats_adjust(psset_bin_stats_t *binstats, edata_t *ps, bool inc) { + size_t mul = inc ? (size_t)1 : (size_t)-1; + + size_t npages = edata_size_get(ps) >> LG_PAGE; + size_t ninactive = edata_nfree_get(ps); + size_t nactive = npages - ninactive; + binstats->npageslabs += mul * 1; + binstats->nactive += mul * nactive; + binstats->ninactive += mul * ninactive; +} + +static void +psset_edata_heap_remove(psset_t *psset, pszind_t pind, edata_t *ps) { + edata_heap_remove(&psset->pageslabs[pind], ps); + psset_bin_stats_adjust(&psset->slab_stats[pind], ps, /* inc */ false); +} + +static void +psset_edata_heap_insert(psset_t *psset, pszind_t pind, edata_t *ps) { + edata_heap_insert(&psset->pageslabs[pind], ps); + psset_bin_stats_adjust(&psset->slab_stats[pind], ps, /* inc */ true); } JEMALLOC_ALWAYS_INLINE void @@ -46,7 +88,8 @@ psset_recycle_extract(psset_t *psset, size_t size) { if (ret == NULL) { return NULL; } - edata_heap_remove(&psset->pageslabs[ret_ind], ret); + + psset_edata_heap_remove(psset, ret_ind, ret); if (edata_heap_empty(&psset->pageslabs[ret_ind])) { bitmap_set(psset->bitmap, &psset_bitmap_info, ret_ind); } @@ -67,7 +110,7 @@ psset_insert(psset_t *psset, edata_t *ps, size_t largest_range) { if (edata_heap_empty(&psset->pageslabs[pind])) { bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind); } - edata_heap_insert(&psset->pageslabs[pind], ps); + psset_edata_heap_insert(psset, pind, ps); } /* @@ -120,6 +163,9 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, EXTENT_NOT_HEAD); edata_ps_set(r_edata, ps); fb_set_range(ps_fb, ps_npages, begin, npages); + edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) - npages)); + /* The pageslab isn't in a bin, so no bin stats need to change. */ + /* * OK, we've got to put the pageslab back. First we have to figure out * where, though; we've only checked run sizes before the pageslab we @@ -144,7 +190,10 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata, start = begin + len; } edata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range); - if (largest_unchosen_range != 0) { + if (largest_unchosen_range == 0) { + psset_bin_stats_adjust(&psset->full_slab_stats, ps, + /* inc */ true); + } else { psset_insert(psset, ps, largest_unchosen_range); } } @@ -164,8 +213,8 @@ psset_alloc_new(psset_t *psset, edata_t *ps, edata_t *r_edata, size_t size) { fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap; size_t ps_npages = edata_size_get(ps) >> LG_PAGE; assert(fb_empty(ps_fb, ps_npages)); - assert(ps_npages >= (size >> LG_PAGE)); + edata_nfree_set(ps, (uint32_t)ps_npages); psset_ps_alloc_insert(psset, ps, r_edata, size); } @@ -177,6 +226,11 @@ psset_dalloc(psset_t *psset, edata_t *edata) { edata_t *ps = edata_ps_get(edata); fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap; size_t ps_old_longest_free_range = edata_longest_free_range_get(ps); + pszind_t old_pind = SC_NPSIZES; + if (ps_old_longest_free_range != 0) { + old_pind = sz_psz2ind(sz_psz_quantize_floor( + ps_old_longest_free_range << LG_PAGE)); + } size_t ps_npages = edata_size_get(ps) >> LG_PAGE; size_t begin = @@ -184,6 +238,23 @@ psset_dalloc(psset_t *psset, edata_t *edata) { >> LG_PAGE; size_t len = edata_size_get(edata) >> LG_PAGE; fb_unset_range(ps_fb, ps_npages, begin, len); + if (ps_old_longest_free_range == 0) { + /* We were in the (imaginary) full bin; update stats for it. */ + psset_bin_stats_adjust(&psset->full_slab_stats, ps, + /* inc */ false); + } else { + /* + * The edata is still in the bin, need to update its + * contribution. + */ + psset->slab_stats[old_pind].nactive -= len; + psset->slab_stats[old_pind].ninactive += len; + } + /* + * Note that we want to do this after the stats updates, since if it was + * full it psset_bin_stats_adjust would have looked at the old version. + */ + edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) + len)); /* We might have just created a new, larger range. */ size_t new_begin = (size_t)(fb_fls(ps_fb, ps_npages, begin) + 1); @@ -215,9 +286,7 @@ psset_dalloc(psset_t *psset, edata_t *edata) { * and the issue becomes moot). */ if (ps_old_longest_free_range > 0) { - pszind_t old_pind = sz_psz2ind(sz_psz_quantize_floor( - ps_old_longest_free_range<< LG_PAGE)); - edata_heap_remove(&psset->pageslabs[old_pind], ps); + psset_edata_heap_remove(psset, old_pind, ps); if (edata_heap_empty(&psset->pageslabs[old_pind])) { bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)old_pind); @@ -234,6 +303,6 @@ psset_dalloc(psset_t *psset, edata_t *edata) { bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)new_pind); } - edata_heap_insert(&psset->pageslabs[new_pind], ps); + psset_edata_heap_insert(psset, new_pind, ps); return NULL; } diff --git a/test/unit/psset.c b/test/unit/psset.c index 8a5090d3..0bc4460f 100644 --- a/test/unit/psset.c +++ b/test/unit/psset.c @@ -295,6 +295,81 @@ TEST_BEGIN(test_multi_pageslab) { } TEST_END +static void +stats_expect_empty(psset_bin_stats_t *stats) { + assert_zu_eq(0, stats->npageslabs, + "Supposedly empty bin had positive npageslabs"); + expect_zu_eq(0, stats->nactive, "Unexpected nonempty bin" + "Supposedly empty bin had positive nactive"); + expect_zu_eq(0, stats->ninactive, "Unexpected nonempty bin" + "Supposedly empty bin had positive ninactive"); +} + +static void +stats_expect(psset_t *psset, size_t nactive) { + if (nactive == PAGESLAB_PAGES) { + expect_zu_eq(1, psset->full_slab_stats.npageslabs, + "Expected a full slab"); + expect_zu_eq(PAGESLAB_PAGES, psset->full_slab_stats.nactive, + "Should have exactly filled the bin"); + expect_zu_eq(0, psset->full_slab_stats.ninactive, + "Should never have inactive pages in a full slab"); + } else { + stats_expect_empty(&psset->full_slab_stats); + } + size_t ninactive = PAGESLAB_PAGES - nactive; + pszind_t nonempty_pind = PSSET_NPSIZES; + if (ninactive != 0 && ninactive < PAGESLAB_PAGES) { + nonempty_pind = sz_psz2ind(sz_psz_quantize_floor( + ninactive << LG_PAGE)); + } + for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { + if (i == nonempty_pind) { + assert_zu_eq(1, psset->slab_stats[i].npageslabs, + "Should have found a slab"); + expect_zu_eq(nactive, psset->slab_stats[i].nactive, + "Mismatch in active pages"); + expect_zu_eq(ninactive, psset->slab_stats[i].ninactive, + "Mismatch in inactive pages"); + } else { + stats_expect_empty(&psset->slab_stats[i]); + } + } +} + +TEST_BEGIN(test_stats) { + bool err; + edata_t pageslab; + memset(&pageslab, 0, sizeof(pageslab)); + edata_t alloc[PAGESLAB_PAGES]; + + edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE, + /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active, + /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA, + EXTENT_IS_HEAD); + + psset_t psset; + psset_init(&psset); + stats_expect(&psset, 0); + + edata_init_test(&alloc[0]); + psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE); + for (size_t i = 1; i < PAGESLAB_PAGES; i++) { + stats_expect(&psset, i); + edata_init_test(&alloc[i]); + err = psset_alloc_reuse(&psset, &alloc[i], PAGE); + expect_false(err, "Nonempty psset failed page allocation."); + } + stats_expect(&psset, PAGESLAB_PAGES); + for (ssize_t i = PAGESLAB_PAGES - 1; i >= 0; i--) { + edata_t *ps = psset_dalloc(&psset, &alloc[i]); + expect_true((ps == NULL) == (i != 0), + "psset_dalloc should only evict a slab on the last free"); + stats_expect(&psset, i); + } +} +TEST_END + int main(void) { return test_no_reentrancy( @@ -302,5 +377,6 @@ main(void) { test_fill, test_reuse, test_evict, - test_multi_pageslab); + test_multi_pageslab, + test_stats); }