psset: keep aggregate stats.

This will let us quickly query these stats to make purging decisions quickly.

include/jemalloc/internal/psset.h

@@ -8,9 +8,6 @@
  * a collection of page-slabs (the intent being that they are backed by
  * hugepages, or at least could be), and handles allocation and deallocation
  * requests.
- *
- * It has the same synchronization guarantees as the eset; stats queries don't
- * need any external synchronization, everything else does.
  */
 
 /*
@@ -60,6 +57,12 @@ struct psset_s {
 	 */
 	hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
 	bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
+	/*
+	 * The sum of all bin stats in stats.  This lets us quickly answer
+	 * queries for the number of dirty, active, and retained pages in the
+	 * entire set.
+	 */
+	psset_bin_stats_t merged_stats;
 	psset_stats_t stats;
 	/*
 	 * Slabs with no active allocations, but which are allowed to serve new
@@ -92,4 +95,19 @@ hpdata_t *psset_pick_hugify(psset_t *psset);
 void psset_insert(psset_t *psset, hpdata_t *ps);
 void psset_remove(psset_t *psset, hpdata_t *ps);
 
+static inline size_t
+psset_npageslabs(psset_t *psset) {
+	return psset->merged_stats.npageslabs;
+}
+
+static inline size_t
+psset_nactive(psset_t *psset) {
+	return psset->merged_stats.nactive;
+}
+
+static inline size_t
+psset_ndirty(psset_t *psset) {
+	return psset->merged_stats.ndirty;
+}
+
 #endif /* JEMALLOC_INTERNAL_PSSET_H */

src/psset.c

@@ -14,6 +14,7 @@ psset_init(psset_t *psset) {
 		hpdata_age_heap_new(&psset->pageslabs[i]);
 	}
 	bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
+	memset(&psset->merged_stats, 0, sizeof(psset->merged_stats));
 	memset(&psset->stats, 0, sizeof(psset->stats));
 	hpdata_empty_list_init(&psset->empty);
 	hpdata_purge_list_init(&psset->to_purge);
@@ -52,23 +53,48 @@ psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) {
  * ensure we don't miss any heap modification operations.
  */
 JEMALLOC_ALWAYS_INLINE void
-psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps,
+psset_bin_stats_insert_remove(psset_t *psset, psset_bin_stats_t *binstats,
-    bool insert) {
+    hpdata_t *ps, bool insert) {
 	size_t mul = insert ? (size_t)1 : (size_t)-1;
 	size_t huge_idx = (size_t)hpdata_huge_get(ps);
+
 	binstats[huge_idx].npageslabs += mul * 1;
 	binstats[huge_idx].nactive += mul * hpdata_nactive_get(ps);
 	binstats[huge_idx].ndirty += mul * hpdata_ndirty_get(ps);
+
+	psset->merged_stats.npageslabs += mul * 1;
+	psset->merged_stats.nactive += mul * hpdata_nactive_get(ps);
+	psset->merged_stats.ndirty += mul * hpdata_ndirty_get(ps);
+
+	if (config_debug) {
+		psset_bin_stats_t check_stats = {0};
+		for (size_t huge = 0; huge <= 1; huge++) {
+			psset_bin_stats_accum(&check_stats,
+			    &psset->stats.full_slabs[huge]);
+			psset_bin_stats_accum(&check_stats,
+			    &psset->stats.empty_slabs[huge]);
+			for (pszind_t pind = 0; pind < PSSET_NPSIZES; pind++) {
+				psset_bin_stats_accum(&check_stats,
+				    &psset->stats.nonfull_slabs[pind][huge]);
+			}
+		}
+		assert(psset->merged_stats.npageslabs
+		    == check_stats.npageslabs);
+		assert(psset->merged_stats.nactive == check_stats.nactive);
+		assert(psset->merged_stats.ndirty == check_stats.ndirty);
+	}
 }
 
 static void
-psset_bin_stats_insert(psset_bin_stats_t *binstats, hpdata_t *ps) {
+psset_bin_stats_insert(psset_t *psset, psset_bin_stats_t *binstats,
-	psset_bin_stats_insert_remove(binstats, ps, true);
+    hpdata_t *ps) {
+	psset_bin_stats_insert_remove(psset, binstats, ps, true);
 }
 
 static void
-psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) {
+psset_bin_stats_remove(psset_t *psset, psset_bin_stats_t *binstats,
-	psset_bin_stats_insert_remove(binstats, ps, false);
+    hpdata_t *ps) {
+	psset_bin_stats_insert_remove(psset, binstats, ps, false);
 }
 
 static void
@@ -90,9 +116,9 @@ psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
 static void
 psset_stats_insert(psset_t* psset, hpdata_t *ps) {
 	if (hpdata_empty(ps)) {
-		psset_bin_stats_insert(psset->stats.empty_slabs, ps);
+		psset_bin_stats_insert(psset, psset->stats.empty_slabs, ps);
 	} else if (hpdata_full(ps)) {
-		psset_bin_stats_insert(psset->stats.full_slabs, ps);
+		psset_bin_stats_insert(psset, psset->stats.full_slabs, ps);
 	} else {
 		size_t longest_free_range = hpdata_longest_free_range_get(ps);
 
@@ -100,16 +126,17 @@ psset_stats_insert(psset_t* psset, hpdata_t *ps) {
 		    longest_free_range << LG_PAGE));
 		assert(pind < PSSET_NPSIZES);
 
-		psset_bin_stats_insert(psset->stats.nonfull_slabs[pind], ps);
+		psset_bin_stats_insert(psset, psset->stats.nonfull_slabs[pind],
+		    ps);
 	}
 }
 
 static void
 psset_stats_remove(psset_t *psset, hpdata_t *ps) {
 	if (hpdata_empty(ps)) {
-		psset_bin_stats_remove(psset->stats.empty_slabs, ps);
+		psset_bin_stats_remove(psset, psset->stats.empty_slabs, ps);
 	} else if (hpdata_full(ps)) {
-		psset_bin_stats_remove(psset->stats.full_slabs, ps);
+		psset_bin_stats_remove(psset, psset->stats.full_slabs, ps);
 	} else {
 		size_t longest_free_range = hpdata_longest_free_range_get(ps);
 
@@ -117,7 +144,8 @@ psset_stats_remove(psset_t *psset, hpdata_t *ps) {
 		    longest_free_range << LG_PAGE));
 		assert(pind < PSSET_NPSIZES);
 
-		psset_bin_stats_remove(psset->stats.nonfull_slabs[pind], ps);
+		psset_bin_stats_remove(psset, psset->stats.nonfull_slabs[pind],
+		    ps);
 	}
 }
 

test/unit/psset.c

@@ -374,6 +374,7 @@ stats_expect(psset_t *psset, size_t nactive) {
 			stats_expect_empty(&psset->stats.nonfull_slabs[i][0]);
 		}
 	}
+	expect_zu_eq(nactive, psset_nactive(psset), "");
 }
 
 TEST_BEGIN(test_stats) {