psset: Add stats

include/jemalloc/internal/edata.h

@@ -218,13 +218,17 @@ struct edata_s {
 		 */
 		edata_t *ps;
 		/*
-		 * If this edata *is* a pageslab, then it has some longest free
-		 * range in it.  Track it.
+		 * If this edata *is* a pageslab, then we cache some useful
+		 * information about its associated bitmap.
 		 */
 		struct {
+			/*
+			 * The longest free range a pageslab contains determines
+			 * the heap it lives in.  If we know that it didn't
+			 * change after an operation, we can avoid moving it
+			 * between heaps.
+			 */
 			uint32_t longest_free_range;
-			/* Not yet tracked. */
-			/* uint32_t longest_free_range_pos; */
 		};
 	};
 

include/jemalloc/internal/psset.h

@@ -21,6 +21,16 @@
  */
 #define PSSET_NPSIZES 64
 
+typedef struct psset_bin_stats_s psset_bin_stats_t;
+struct psset_bin_stats_s {
+	/* How many pageslabs are in this bin? */
+	size_t npageslabs;
+	/* Of them, how many pages are active? */
+	size_t nactive;
+	/* How many are inactive? */
+	size_t ninactive;
+};
+
 typedef struct psset_s psset_t;
 struct psset_s {
 	/*
@@ -29,6 +39,12 @@ struct psset_s {
 	 */
 	edata_heap_t pageslabs[PSSET_NPSIZES];
 	bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
+	/*
+	 * Full slabs don't live in any edata heap.  But we still track their
+	 * stats.
+	 */
+	psset_bin_stats_t full_slab_stats;
+	psset_bin_stats_t slab_stats[PSSET_NPSIZES];
 };
 
 void psset_init(psset_t *psset);

src/psset.c

@@ -14,6 +14,48 @@ psset_init(psset_t *psset) {
 		edata_heap_new(&psset->pageslabs[i]);
 	}
 	bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
+	psset->full_slab_stats.npageslabs = 0;
+	psset->full_slab_stats.nactive = 0;
+	psset->full_slab_stats.ninactive = 0;
+	for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
+		psset->slab_stats[i].npageslabs = 0;
+		psset->slab_stats[i].nactive = 0;
+		psset->slab_stats[i].ninactive = 0;
+	}
+}
+
+/*
+ * The stats maintenance strategy is simple, but not necessarily obvious.
+ * edata_nfree and the bitmap must remain consistent at all times.  If they
+ * change while an edata is within an edata_heap (or full), then the associated
+ * stats bin (or the full bin) must also change.  If they change while not in a
+ * bin (say, in between extraction and reinsertion), then the bin stats need not
+ * change.  If a pageslab is removed from a bin (or becomes nonfull), it should
+ * no longer contribute to that bin's stats (or the full stats).  These help
+ * ensure we don't miss any heap modification operations.
+ */
+JEMALLOC_ALWAYS_INLINE void
+psset_bin_stats_adjust(psset_bin_stats_t *binstats, edata_t *ps, bool inc) {
+	size_t mul = inc ? (size_t)1 : (size_t)-1;
+
+	size_t npages = edata_size_get(ps) >> LG_PAGE;
+	size_t ninactive = edata_nfree_get(ps);
+	size_t nactive = npages - ninactive;
+	binstats->npageslabs += mul * 1;
+	binstats->nactive += mul * nactive;
+	binstats->ninactive += mul * ninactive;
+}
+
+static void
+psset_edata_heap_remove(psset_t *psset, pszind_t pind, edata_t *ps) {
+	edata_heap_remove(&psset->pageslabs[pind], ps);
+	psset_bin_stats_adjust(&psset->slab_stats[pind], ps, /* inc */ false);
+}
+
+static void
+psset_edata_heap_insert(psset_t *psset, pszind_t pind, edata_t *ps) {
+	edata_heap_insert(&psset->pageslabs[pind], ps);
+	psset_bin_stats_adjust(&psset->slab_stats[pind], ps, /* inc */ true);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -46,7 +88,8 @@ psset_recycle_extract(psset_t *psset, size_t size) {
 	if (ret == NULL) {
 		return NULL;
 	}
-	edata_heap_remove(&psset->pageslabs[ret_ind], ret);
+
+	psset_edata_heap_remove(psset, ret_ind, ret);
 	if (edata_heap_empty(&psset->pageslabs[ret_ind])) {
 		bitmap_set(psset->bitmap, &psset_bitmap_info, ret_ind);
 	}
@@ -67,7 +110,7 @@ psset_insert(psset_t *psset, edata_t *ps, size_t largest_range) {
 	if (edata_heap_empty(&psset->pageslabs[pind])) {
 		bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
 	}
-	edata_heap_insert(&psset->pageslabs[pind], ps);
+	psset_edata_heap_insert(psset, pind, ps);
 }
 
 /*
@@ -120,6 +163,9 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
 	    EXTENT_NOT_HEAD);
 	edata_ps_set(r_edata, ps);
 	fb_set_range(ps_fb, ps_npages, begin, npages);
+	edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) - npages));
+	/* The pageslab isn't in a bin, so no bin stats need to change. */
+
 	/*
 	 * OK, we've got to put the pageslab back.  First we have to figure out
 	 * where, though; we've only checked run sizes before the pageslab we
@@ -144,7 +190,10 @@ psset_ps_alloc_insert(psset_t *psset, edata_t *ps, edata_t *r_edata,
 		start = begin + len;
 	}
 	edata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range);
-	if (largest_unchosen_range != 0) {
+	if (largest_unchosen_range == 0) {
+		psset_bin_stats_adjust(&psset->full_slab_stats, ps,
+		    /* inc */ true);
+	} else {
 		psset_insert(psset, ps, largest_unchosen_range);
 	}
 }
@@ -164,8 +213,8 @@ psset_alloc_new(psset_t *psset, edata_t *ps, edata_t *r_edata, size_t size) {
 	fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
 	size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
 	assert(fb_empty(ps_fb, ps_npages));
-
 	assert(ps_npages >= (size >> LG_PAGE));
+	edata_nfree_set(ps, (uint32_t)ps_npages);
 	psset_ps_alloc_insert(psset, ps, r_edata, size);
 }
 
@@ -177,6 +226,11 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
 	edata_t *ps = edata_ps_get(edata);
 	fb_group_t *ps_fb = edata_slab_data_get(ps)->bitmap;
 	size_t ps_old_longest_free_range = edata_longest_free_range_get(ps);
+	pszind_t old_pind = SC_NPSIZES;
+	if (ps_old_longest_free_range != 0) {
+		old_pind = sz_psz2ind(sz_psz_quantize_floor(
+		    ps_old_longest_free_range << LG_PAGE));
+	}
 
 	size_t ps_npages = edata_size_get(ps) >> LG_PAGE;
 	size_t begin =
@@ -184,6 +238,23 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
 	    >> LG_PAGE;
 	size_t len = edata_size_get(edata) >> LG_PAGE;
 	fb_unset_range(ps_fb, ps_npages, begin, len);
+	if (ps_old_longest_free_range == 0) {
+		/* We were in the (imaginary) full bin; update stats for it. */
+		psset_bin_stats_adjust(&psset->full_slab_stats, ps,
+		    /* inc */ false);
+	} else {
+		/*
+		 * The edata is still in the bin, need to update its
+		 * contribution.
+		 */
+		psset->slab_stats[old_pind].nactive -= len;
+		psset->slab_stats[old_pind].ninactive += len;
+	}
+	/*
+	 * Note that we want to do this after the stats updates, since if it was
+	 * full it psset_bin_stats_adjust would have looked at the old version.
+	 */
+	edata_nfree_set(ps, (uint32_t)(edata_nfree_get(ps) + len));
 
 	/* We might have just created a new, larger range. */
 	size_t new_begin = (size_t)(fb_fls(ps_fb, ps_npages, begin) + 1);
@@ -215,9 +286,7 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
 	 * and the issue becomes moot).
 	 */
 	if (ps_old_longest_free_range > 0) {
-		pszind_t old_pind = sz_psz2ind(sz_psz_quantize_floor(
-		    ps_old_longest_free_range<< LG_PAGE));
-		edata_heap_remove(&psset->pageslabs[old_pind], ps);
+		psset_edata_heap_remove(psset, old_pind, ps);
 		if (edata_heap_empty(&psset->pageslabs[old_pind])) {
 			bitmap_set(psset->bitmap, &psset_bitmap_info,
 			    (size_t)old_pind);
@@ -234,6 +303,6 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
 		bitmap_unset(psset->bitmap, &psset_bitmap_info,
 		    (size_t)new_pind);
 	}
-	edata_heap_insert(&psset->pageslabs[new_pind], ps);
+	psset_edata_heap_insert(psset, new_pind, ps);
 	return NULL;
 }

test/unit/psset.c

@@ -295,6 +295,81 @@ TEST_BEGIN(test_multi_pageslab) {
 }
 TEST_END
 
+static void
+stats_expect_empty(psset_bin_stats_t *stats) {
+	assert_zu_eq(0, stats->npageslabs,
+	    "Supposedly empty bin had positive npageslabs");
+	expect_zu_eq(0, stats->nactive, "Unexpected nonempty bin"
+	    "Supposedly empty bin had positive nactive");
+	expect_zu_eq(0, stats->ninactive, "Unexpected nonempty bin"
+	    "Supposedly empty bin had positive ninactive");
+}
+
+static void
+stats_expect(psset_t *psset, size_t nactive) {
+	if (nactive == PAGESLAB_PAGES) {
+		expect_zu_eq(1, psset->full_slab_stats.npageslabs,
+		    "Expected a full slab");
+		expect_zu_eq(PAGESLAB_PAGES, psset->full_slab_stats.nactive,
+		    "Should have exactly filled the bin");
+		expect_zu_eq(0, psset->full_slab_stats.ninactive,
+		    "Should never have inactive pages in a full slab");
+	} else {
+		stats_expect_empty(&psset->full_slab_stats);
+	}
+	size_t ninactive = PAGESLAB_PAGES - nactive;
+	pszind_t nonempty_pind = PSSET_NPSIZES;
+	if (ninactive != 0 && ninactive < PAGESLAB_PAGES) {
+		nonempty_pind = sz_psz2ind(sz_psz_quantize_floor(
+		    ninactive << LG_PAGE));
+	}
+	for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
+		if (i == nonempty_pind) {
+			assert_zu_eq(1, psset->slab_stats[i].npageslabs,
+			    "Should have found a slab");
+			expect_zu_eq(nactive, psset->slab_stats[i].nactive,
+			    "Mismatch in active pages");
+			expect_zu_eq(ninactive, psset->slab_stats[i].ninactive,
+			    "Mismatch in inactive pages");
+		} else {
+			stats_expect_empty(&psset->slab_stats[i]);
+		}
+	}
+}
+
+TEST_BEGIN(test_stats) {
+	bool err;
+	edata_t pageslab;
+	memset(&pageslab, 0, sizeof(pageslab));
+	edata_t alloc[PAGESLAB_PAGES];
+
+	edata_init(&pageslab, /* arena_ind */ 0, PAGESLAB_ADDR, PAGESLAB_SIZE,
+	    /* slab */ true, SC_NSIZES, PAGESLAB_SN, extent_state_active,
+	    /* zeroed */ false, /* comitted */ true, EXTENT_PAI_HPA,
+	    EXTENT_IS_HEAD);
+
+	psset_t psset;
+	psset_init(&psset);
+	stats_expect(&psset, 0);
+
+	edata_init_test(&alloc[0]);
+	psset_alloc_new(&psset, &pageslab, &alloc[0], PAGE);
+	for (size_t i = 1; i < PAGESLAB_PAGES; i++) {
+		stats_expect(&psset, i);
+		edata_init_test(&alloc[i]);
+		err = psset_alloc_reuse(&psset, &alloc[i], PAGE);
+		expect_false(err, "Nonempty psset failed page allocation.");
+	}
+	stats_expect(&psset, PAGESLAB_PAGES);
+	for (ssize_t i = PAGESLAB_PAGES - 1; i >= 0; i--) {
+		edata_t *ps = psset_dalloc(&psset, &alloc[i]);
+		expect_true((ps == NULL) == (i != 0),
+		    "psset_dalloc should only evict a slab on the last free");
+		stats_expect(&psset, i);
+	}
+}
+TEST_END
+
 int
 main(void) {
 	return test_no_reentrancy(
@@ -302,5 +377,6 @@ main(void) {
 	    test_fill,
 	    test_reuse,
 	    test_evict,
-	    test_multi_pageslab);
+	    test_multi_pageslab,
+	    test_stats);
 }