psset: Purge empty slabs first.

These are particularly good candidates for purging (listed in the diff).

include/jemalloc/internal/psset.h

@@ -25,6 +25,9 @@
  * index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind +
  * 1).  This lets us implement a preference for purging non-hugified hpdatas
  * among similarly-dirty ones.
+ * We reserve the last two indices for empty slabs, in that case purging
+ * hugified ones (which are definitionally all waste) before non-hugified ones
+ * (i.e. reversing the order).
  */
 #define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES)
 
@@ -78,7 +81,11 @@ struct psset_s {
 	 * allocations.
 	 */
 	hpdata_empty_list_t empty;
-	/* Slabs which are available to be purged, ordered by purge level. */
+	/*
+	 * Slabs which are available to be purged, ordered by how much we want
+	 * to purge them (with later indices indicating slabs we want to purge
+	 * more).
+	 */
 	hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS];
 	/* Bitmap for which set bits correspond to non-empty purge lists. */
 	fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)];

src/psset.c

@@ -201,11 +201,32 @@ psset_purge_list_ind(hpdata_t *ps) {
 	size_t ndirty = hpdata_ndirty_get(ps);
 	/* Shouldn't have something with no dirty pages purgeable. */
 	assert(ndirty > 0);
+	/*
+	 * Higher indices correspond to lists we'd like to purge earlier; make
+	 * the two highest indices correspond to empty lists, which we attempt
+	 * to purge before purging any non-empty list.  This has two advantages:
+	 * - Empty page slabs are the least likely to get reused (we'll only
+	 *   pick them for an allocation if we have no other choice).
+	 * - Empty page slabs can purge every dirty page they contain in a
+	 *   single call, which is not usually the case.
+	 *
+	 * We purge hugeified empty slabs before nonhugeified ones, on the basis
+	 * that they are fully dirty, while nonhugified slabs might not be, so
+	 * we free up more pages more easily.
+	 */
+	if (hpdata_nactive_get(ps) == 0) {
+		if (hpdata_huge_get(ps)) {
+			return PSSET_NPURGE_LISTS - 1;
+		} else {
+			return PSSET_NPURGE_LISTS - 2;
+		}
+	}
+
 	pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(ndirty << LG_PAGE));
 	/*
-	 * Higher indices correspond to lists we'd like to purge earlier;
+	 * For non-empty slabs, we may reuse them again.  Prefer purging
-	 * increment the index for the nonhugified hpdatas first, so that we'll
+	 * non-hugeified slabs before hugeified ones then, among pages of
-	 * pick them before picking hugified ones.
+	 * similar dirtiness.  We still get some benefit from the hugification.
 	 */
 	return (size_t)pind * 2 + (hpdata_huge_get(ps) ? 0 : 1);
 }
@@ -321,7 +342,7 @@ psset_pick_purge(psset_t *psset) {
 		return NULL;
 	}
 	pszind_t ind = (pszind_t)ind_ssz;
-	assert(ind < PSSET_NPSIZES);
+	assert(ind < PSSET_NPURGE_LISTS);
 	hpdata_t *ps = hpdata_purge_list_first(&psset->to_purge[ind]);
 	assert(ps != NULL);
 	return ps;

test/unit/psset.c

@@ -545,7 +545,7 @@ TEST_END
 TEST_BEGIN(test_purge_prefers_nonhuge) {
 	/*
 	 * All else being equal, we should prefer purging non-huge pages over
-	 * huge ones.
+	 * huge ones for non-empty extents.
 	 */
 
 	/* Nothing magic about this constant. */
@@ -625,6 +625,112 @@ TEST_BEGIN(test_purge_prefers_nonhuge) {
 }
 TEST_END
 
+TEST_BEGIN(test_purge_prefers_empty) {
+	void *ptr;
+
+	psset_t psset;
+	psset_init(&psset);
+
+	hpdata_t hpdata_empty;
+	hpdata_t hpdata_nonempty;
+	hpdata_init(&hpdata_empty, (void *)(10 * HUGEPAGE), 123);
+	psset_insert(&psset, &hpdata_empty);
+	hpdata_init(&hpdata_nonempty, (void *)(11 * HUGEPAGE), 456);
+	psset_insert(&psset, &hpdata_nonempty);
+
+	psset_update_begin(&psset, &hpdata_empty);
+	ptr = hpdata_reserve_alloc(&hpdata_empty, PAGE);
+	expect_ptr_eq(hpdata_addr_get(&hpdata_empty), ptr, "");
+	hpdata_unreserve(&hpdata_empty, ptr, PAGE);
+	hpdata_purge_allowed_set(&hpdata_empty, true);
+	psset_update_end(&psset, &hpdata_empty);
+
+	psset_update_begin(&psset, &hpdata_nonempty);
+	ptr = hpdata_reserve_alloc(&hpdata_nonempty, 10 * PAGE);
+	expect_ptr_eq(hpdata_addr_get(&hpdata_nonempty), ptr, "");
+	hpdata_unreserve(&hpdata_nonempty, ptr, 9 * PAGE);
+	hpdata_purge_allowed_set(&hpdata_nonempty, true);
+	psset_update_end(&psset, &hpdata_nonempty);
+
+	/*
+	 * The nonempty slab has 9 dirty pages, while the empty one has only 1.
+	 * We should still pick the empty one for purging.
+	 */
+	hpdata_t *to_purge = psset_pick_purge(&psset);
+	expect_ptr_eq(&hpdata_empty, to_purge, "");
+}
+TEST_END
+
+TEST_BEGIN(test_purge_prefers_empty_huge) {
+	void *ptr;
+
+	psset_t psset;
+	psset_init(&psset);
+
+	enum {NHP = 10 };
+
+	hpdata_t hpdata_huge[NHP];
+	hpdata_t hpdata_nonhuge[NHP];
+
+	uintptr_t cur_addr = 100 * HUGEPAGE;
+	uint64_t cur_age = 123;
+	for (int i = 0; i < NHP; i++) {
+		hpdata_init(&hpdata_huge[i], (void *)cur_addr, cur_age);
+		cur_addr += HUGEPAGE;
+		cur_age++;
+		psset_insert(&psset, &hpdata_huge[i]);
+
+		hpdata_init(&hpdata_nonhuge[i], (void *)cur_addr, cur_age);
+		cur_addr += HUGEPAGE;
+		cur_age++;
+		psset_insert(&psset, &hpdata_nonhuge[i]);
+
+		/*
+		 * Make the hpdata_huge[i] fully dirty, empty, purgable, and
+		 * huge.
+		 */
+		psset_update_begin(&psset, &hpdata_huge[i]);
+		ptr = hpdata_reserve_alloc(&hpdata_huge[i], HUGEPAGE);
+		expect_ptr_eq(hpdata_addr_get(&hpdata_huge[i]), ptr, "");
+		hpdata_hugify(&hpdata_huge[i]);
+		hpdata_unreserve(&hpdata_huge[i], ptr, HUGEPAGE);
+		hpdata_purge_allowed_set(&hpdata_huge[i], true);
+		psset_update_end(&psset, &hpdata_huge[i]);
+
+		/*
+		 * Make hpdata_nonhuge[i] fully dirty, empty, purgable, and
+		 * non-huge.
+		 */
+		psset_update_begin(&psset, &hpdata_nonhuge[i]);
+		ptr = hpdata_reserve_alloc(&hpdata_nonhuge[i], HUGEPAGE);
+		expect_ptr_eq(hpdata_addr_get(&hpdata_nonhuge[i]), ptr, "");
+		hpdata_unreserve(&hpdata_nonhuge[i], ptr, HUGEPAGE);
+		hpdata_purge_allowed_set(&hpdata_nonhuge[i], true);
+		psset_update_end(&psset, &hpdata_nonhuge[i]);
+	}
+
+	/*
+	 * We have a bunch of empty slabs, half huge, half nonhuge, inserted in
+	 * alternating order.  We should pop all the huge ones before popping
+	 * any of the non-huge ones for purging.
+	 */
+	for (int i = 0; i < NHP; i++) {
+		hpdata_t *to_purge = psset_pick_purge(&psset);
+		expect_ptr_eq(&hpdata_huge[i], to_purge, "");
+		psset_update_begin(&psset, to_purge);
+		hpdata_purge_allowed_set(to_purge, false);
+		psset_update_end(&psset, to_purge);
+	}
+	for (int i = 0; i < NHP; i++) {
+		hpdata_t *to_purge = psset_pick_purge(&psset);
+		expect_ptr_eq(&hpdata_nonhuge[i], to_purge, "");
+		psset_update_begin(&psset, to_purge);
+		hpdata_purge_allowed_set(to_purge, false);
+		psset_update_end(&psset, to_purge);
+	}
+}
+TEST_END
+
 int
 main(void) {
 	return test_no_reentrancy(
@@ -636,5 +742,7 @@ main(void) {
 	    test_stats,
 	    test_oldest_fit,
 	    test_insert_remove,
-	    test_purge_prefers_nonhuge);
+	    test_purge_prefers_nonhuge,
+	    test_purge_prefers_empty,
+	    test_purge_prefers_empty_huge);
 }