Move empty slab tracking to the psset.

We're moving towards a world in which purging decisions are less rigidly
enforced at a single-hugepage level.  In that world, it makes sense to keep
around some hpdatas which are not completely purged, in which case we'll need to
track them.

include/jemalloc/internal/hpa.h

@@ -8,14 +8,6 @@
 
 typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
 struct hpa_shard_nonderived_stats_s {
-	/*
-	 * The number of times we've fully purged a hugepage and evicted it from
-	 * the psset.
-	 *
-	 * Guarded by grow_mtx.
-	 */
-	uint64_t nevictions;
-
 	/*
 	 * The number of times we've purged within a hugepage.
 	 *
@@ -80,15 +72,6 @@ struct hpa_shard_s {
 	 */
 	size_t alloc_max;
 
-	/*
-	 * Slabs currently purged away.  They are hugepage-sized and
-	 * hugepage-aligned, but have had pages_nohuge and pages_purge_forced
-	 * called on them.
-	 *
-	 * Guarded by grow_mtx.
-	 */
-	hpdata_list_t unused_slabs;
-
 	/*
 	 * How many grow operations have occurred.
 	 *

include/jemalloc/internal/hpdata.h

@@ -52,14 +52,17 @@ struct hpdata_s {
 	 */
 	bool h_updating;
 
+	/* Whether or not the hpdata is in a psset. */
+	bool h_in_psset;
+
 	union {
-		/* When nonempty, used by the psset bins. */
+		/* When nonempty (and also nonfull), used by the psset bins. */
 		phn(hpdata_t) ph_link;
 		/*
 		 * When empty (or not corresponding to any hugepage), list
 		 * linkage.
 		 */
-		ql_elm(hpdata_t) ql_link;
+		ql_elm(hpdata_t) ql_link_empty;
 	};
 
 	/* The length of the largest contiguous sequence of inactive pages. */
@@ -82,7 +85,7 @@ struct hpdata_s {
 	fb_group_t touched_pages[FB_NGROUPS(HUGEPAGE_PAGES)];
 };
 
-TYPED_LIST(hpdata_list, hpdata_t, ql_link)
+TYPED_LIST(hpdata_empty_list, hpdata_t, ql_link_empty)
 typedef ph(hpdata_t) hpdata_age_heap_t;
 ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t);
 
@@ -138,6 +141,17 @@ hpdata_updating_set(hpdata_t *hpdata, bool updating) {
 	hpdata->h_updating = updating;
 }
 
+static inline bool
+hpdata_in_psset_get(const hpdata_t *hpdata) {
+	return hpdata->h_in_psset;
+}
+
+static inline void
+hpdata_in_psset_set(hpdata_t *hpdata, bool in_psset) {
+	assert(in_psset != hpdata->h_in_psset);
+	hpdata->h_in_psset = in_psset;
+}
+
 static inline size_t
 hpdata_longest_free_range_get(const hpdata_t *hpdata) {
 	return hpdata->h_longest_free_range;
@@ -208,6 +222,11 @@ hpdata_empty(hpdata_t *hpdata) {
 	return hpdata->h_nactive == 0;
 }
 
+static inline bool
+hpdata_full(hpdata_t *hpdata) {
+	return hpdata->h_nactive == HUGEPAGE_PAGES;
+}
+
 void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
 
 /*

include/jemalloc/internal/psset.h

@@ -35,7 +35,6 @@ struct psset_bin_stats_s {
 
 typedef struct psset_stats_s psset_stats_t;
 struct psset_stats_s {
-
 	/*
 	 * The second index is huge stats; nonfull_slabs[pszind][0] contains
 	 * stats for the non-huge slabs in bucket pszind, while
@@ -44,10 +43,13 @@ struct psset_stats_s {
 	psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2];
 
 	/*
-	 * Full slabs don't live in any edata heap.  But we still track their
+	 * Full slabs don't live in any edata heap, but we still track their
 	 * stats.
 	 */
 	psset_bin_stats_t full_slabs[2];
+
+	/* Empty slabs are similar. */
+	psset_bin_stats_t empty_slabs[2];
 };
 
 typedef struct psset_s psset_t;
@@ -59,6 +61,8 @@ struct psset_s {
 	hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
 	bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
 	psset_stats_t stats;
+	/* Slabs with no active allocations. */
+	hpdata_empty_list_t empty_slabs;
 };
 
 void psset_init(psset_t *psset);
@@ -74,4 +78,7 @@ void psset_update_end(psset_t *psset, hpdata_t *ps);
 /* Analogous to the eset_fit; pick a hpdata to serve the request. */
 hpdata_t *psset_pick_alloc(psset_t *psset, size_t size);
 
+void psset_insert(psset_t *psset, hpdata_t *ps);
+void psset_remove(psset_t *psset, hpdata_t *ps);
+
 #endif /* JEMALLOC_INTERNAL_PSSET_H */