HPA: Add stats for the hpa_shard.

include/jemalloc/internal/arena_externs.h

@@ -28,7 +28,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
     bin_stats_data_t *bstats, arena_stats_large_t *lstats,
-    pac_estats_t *estats);
+    pac_estats_t *estats, hpa_shard_stats_t *hpastats);
 void arena_handle_new_dirty_pages(tsdn_t *tsdn, arena_t *arena);
 #ifdef JEMALLOC_JET
 size_t arena_slab_regind(edata_t *slab, szind_t binind, const void *ptr);

include/jemalloc/internal/ctl.h

@@ -45,6 +45,7 @@ typedef struct ctl_arena_stats_s {
 	bin_stats_data_t bstats[SC_NBINS];
 	arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
 	pac_estats_t estats[SC_NPSIZES];
+	hpa_shard_stats_t hpastats;
 } ctl_arena_stats_t;
 
 typedef struct ctl_stats_s {

include/jemalloc/internal/hpa.h

@@ -29,6 +29,13 @@ struct hpa_s {
 	geom_grow_t geom_grow;
 };
 
+/* Used only by CTL; not actually stored here (i.e., all derived). */
+typedef struct hpa_shard_stats_s hpa_shard_stats_t;
+struct hpa_shard_stats_s {
+	psset_bin_stats_t psset_full_slab_stats;
+	psset_bin_stats_t psset_slab_stats[PSSET_NPSIZES];
+};
+
 typedef struct hpa_shard_s hpa_shard_t;
 struct hpa_shard_s {
 	/*

include/jemalloc/internal/mutex_prof.h

@@ -31,7 +31,9 @@ typedef enum {
     OP(decay_dirty)							\
     OP(decay_muzzy)							\
     OP(base)								\
-    OP(tcache_list)
+    OP(tcache_list)							\
+    OP(hpa_shard)							\
+    OP(hpa_shard_grow)
 
 typedef enum {
 #define OP(mtx) arena_prof_mutex_##mtx,

include/jemalloc/internal/pa.h

@@ -192,7 +192,7 @@ void pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive,
 
 void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
-    size_t *resident);
+    hpa_shard_stats_t *hpa_stats_out, size_t *resident);
 
 /*
  * Reads the PA-owned mutex stats into the output stats array, at the

include/jemalloc/internal/psset.h

@@ -31,6 +31,13 @@ struct psset_bin_stats_s {
 	size_t ninactive;
 };
 
+static inline void
+psset_bin_stats_accum(psset_bin_stats_t *dst, psset_bin_stats_t *src) {
+	dst->npageslabs += src->npageslabs;
+	dst->nactive += src->nactive;
+	dst->ninactive += src->ninactive;
+}
+
 typedef struct psset_s psset_t;
 struct psset_s {
 	/*

include/jemalloc/internal/stats.h

@@ -11,7 +11,8 @@
     OPTION('b',		bins,		true,		false)		\
     OPTION('l',		large,		true,		false)		\
     OPTION('x',		mutex,		true,		false)		\
-    OPTION('e',		extents,	true,		false)
+    OPTION('e',		extents,	true,		false)		\
+    OPTION('h',		hpa,		config_stats,	false)
 
 enum {
 #define OPTION(o, v, d, s) stats_print_option_num_##v,

src/arena.c

@@ -81,7 +81,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
     bin_stats_data_t *bstats, arena_stats_large_t *lstats,
-    pac_estats_t *estats) {
+    pac_estats_t *estats, hpa_shard_stats_t *hpastats) {
 	cassert(config_stats);
 
 	arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms,
@@ -139,7 +139,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	}
 
 	pa_shard_stats_merge(tsdn, &arena->pa_shard, &astats->pa_shard_stats,
-	    estats, &astats->resident);
+	    estats, hpastats, &astats->resident);
 
 	LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
 

src/ctl.c

@@ -216,6 +216,13 @@ CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes)
 CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes)
 CTL_PROTO(stats_arenas_i_extents_j_retained_bytes)
 INDEX_PROTO(stats_arenas_i_extents_j)
+CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs)
+CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive)
+CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive)
+INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j)
+CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs)
+CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive)
+CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_ninactive)
 CTL_PROTO(stats_arenas_i_nthreads)
 CTL_PROTO(stats_arenas_i_uptime)
 CTL_PROTO(stats_arenas_i_dss)
@@ -584,6 +591,41 @@ MUTEX_PROF_ARENA_MUTEXES
 #undef OP
 };
 
+static const ctl_named_node_t stats_arenas_i_hpa_shard_full_slabs_node[] = {
+	{NAME("npageslabs"),
+		CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs)},
+	{NAME("nactive"),
+		CTL(stats_arenas_i_hpa_shard_full_slabs_nactive)},
+	{NAME("ninactive"),
+		CTL(stats_arenas_i_hpa_shard_full_slabs_ninactive)}
+};
+
+static const ctl_named_node_t stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = {
+	{NAME("npageslabs"),
+		CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs)},
+	{NAME("nactive"),
+		CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive)},
+	{NAME("ninactive"),
+		CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive)}
+};
+
+static const ctl_named_node_t super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = {
+	{NAME(""),
+		CHILD(named, stats_arenas_i_hpa_shard_nonfull_slabs_j)}
+};
+
+static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] =
+{
+	{INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)}
+};
+
+static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
+	{NAME("full_slabs"),	CHILD(named,
+	    stats_arenas_i_hpa_shard_full_slabs)},
+	{NAME("nonfull_slabs"),	CHILD(indexed,
+	    stats_arenas_i_hpa_shard_nonfull_slabs)}
+};
+
 static const ctl_named_node_t stats_arenas_i_node[] = {
 	{NAME("nthreads"),	CTL(stats_arenas_i_nthreads)},
 	{NAME("uptime"),	CTL(stats_arenas_i_uptime)},
@@ -613,7 +655,8 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
 	{NAME("bins"),		CHILD(indexed, stats_arenas_i_bins)},
 	{NAME("lextents"),	CHILD(indexed, stats_arenas_i_lextents)},
 	{NAME("extents"),	CHILD(indexed, stats_arenas_i_extents)},
-	{NAME("mutexes"),	CHILD(named, stats_arenas_i_mutexes)}
+	{NAME("mutexes"),	CHILD(named, stats_arenas_i_mutexes)},
+	{NAME("hpa_shard"),	CHILD(named, stats_arenas_i_hpa_shard)}
 };
 static const ctl_named_node_t super_stats_arenas_i_node[] = {
 	{NAME(""),		CHILD(named, stats_arenas_i)}
@@ -844,6 +887,8 @@ ctl_arena_clear(ctl_arena_t *ctl_arena) {
 		    sizeof(arena_stats_large_t));
 		memset(ctl_arena->astats->estats, 0, SC_NPSIZES *
 		    sizeof(pac_estats_t));
+		memset(&ctl_arena->astats->hpastats, 0,
+		    sizeof(hpa_shard_stats_t));
 	}
 }
 
@@ -857,7 +902,8 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
 		    &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,
 		    &ctl_arena->pdirty, &ctl_arena->pmuzzy,
 		    &ctl_arena->astats->astats, ctl_arena->astats->bstats,
-		    ctl_arena->astats->lstats, ctl_arena->astats->estats);
+		    ctl_arena->astats->lstats, ctl_arena->astats->estats,
+		    &ctl_arena->astats->hpastats);
 
 		for (i = 0; i < SC_NBINS; i++) {
 			bin_stats_t *bstats =
@@ -1033,6 +1079,16 @@ MUTEX_PROF_ARENA_MUTEXES
 			sdstats->estats[i].retained_bytes
 			    += astats->estats[i].retained_bytes;
 		}
+
+		/* Merge HPA stats. */
+		psset_bin_stats_accum(&sdstats->hpastats.psset_full_slab_stats,
+		    &astats->hpastats.psset_full_slab_stats);
+		for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
+			psset_bin_stats_accum(
+			    &sdstats->hpastats.psset_slab_stats[i],
+			    &astats->hpastats.psset_slab_stats[i]);
+		}
+
 	}
 }
 
@@ -3256,6 +3312,33 @@ stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib,
 	return super_stats_arenas_i_extents_j_node;
 }
 
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs,
+    arenas_i(mib[2])->astats->hpastats.psset_full_slab_stats.npageslabs,
+    size_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive,
+    arenas_i(mib[2])->astats->hpastats.psset_full_slab_stats.nactive, size_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive,
+    arenas_i(mib[2])->astats->hpastats.psset_full_slab_stats.ninactive, size_t);
+
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs,
+    arenas_i(mib[2])->astats->hpastats.psset_slab_stats[mib[5]].npageslabs,
+    size_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive,
+    arenas_i(mib[2])->astats->hpastats.psset_slab_stats[mib[5]].nactive,
+    size_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive,
+    arenas_i(mib[2])->astats->hpastats.psset_slab_stats[mib[5]].ninactive,
+    size_t);
+
+static const ctl_named_node_t *
+stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib,
+    size_t miblen, size_t j) {
+	if (j >= PSSET_NPSIZES) {
+		return NULL;
+	}
+	return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node;
+}
+
 static bool
 ctl_arenas_i_verify(size_t i) {
 	size_t a = arenas_i2a_impl(i, true, true);

src/hpa.c

@@ -258,7 +258,6 @@ hpa_from_pai(pai_t *self) {
 static edata_t *
 hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
     size_t alignment, bool zero) {
-
 	assert((size & PAGE_MASK) == 0);
 	hpa_shard_t *shard = hpa_from_pai(self);
 	/* We don't handle alignment or zeroing for now. */

src/pa_extra.c

@@ -76,7 +76,7 @@ pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, size_t *ndirty,
 void
 pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
-    size_t *resident) {
+    hpa_shard_stats_t *hpa_stats_out, size_t *resident) {
 	cassert(config_stats);
 
 	pa_shard_stats_out->pac_stats.retained +=
@@ -138,6 +138,18 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
 		estats_out[i].muzzy_bytes = muzzy_bytes;
 		estats_out[i].retained_bytes = retained_bytes;
 	}
+
+	if (shard->ever_used_hpa) {
+		malloc_mutex_lock(tsdn, &shard->hpa_shard.mtx);
+		psset_bin_stats_accum(&hpa_stats_out->psset_full_slab_stats,
+		    &shard->hpa_shard.psset.full_slab_stats);
+		for (pszind_t i = 0; i < PSSET_NPSIZES; i++) {
+			psset_bin_stats_accum(
+			    &hpa_stats_out->psset_slab_stats[i],
+			    &shard->hpa_shard.psset.slab_stats[i]);
+		}
+		malloc_mutex_unlock(tsdn, &shard->hpa_shard.mtx);
+	}
 }
 
 static void
@@ -163,4 +175,12 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
 	    &shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty);
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 	    &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy);
+
+	if (shard->ever_used_hpa) {
+		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
+		    &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard);
+		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
+		    &shard->hpa_shard.grow_mtx,
+		    arena_prof_mutex_hpa_shard_grow);
+	}
 }

src/stats.c

@@ -43,6 +43,16 @@ const char *arena_mutex_names[mutex_prof_num_arena_mutexes] = {
 	xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0);		\
 } while (0)
 
+#define CTL_M2_M5_GET(n, i, j, v, t) do {				\
+	size_t mib[CTL_MAX_DEPTH];					\
+	size_t miblen = sizeof(mib) / sizeof(size_t);			\
+	size_t sz = sizeof(t);						\
+	xmallctlnametomib(n, mib, &miblen);				\
+	mib[2] = (i);							\
+	mib[5] = (j);							\
+	xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0);		\
+} while (0)
+
 /******************************************************************************/
 /* Data. */
 
@@ -650,6 +660,87 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 	}
 }
 
+static void
+stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i) {
+	emitter_row_t header_row;
+	emitter_row_init(&header_row);
+	emitter_row_t row;
+	emitter_row_init(&row);
+
+	size_t npageslabs;
+	size_t nactive;
+	size_t ninactive;
+
+	CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs",
+	    i, &npageslabs, size_t);
+	CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive",
+	    i, &nactive, size_t);
+	CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ninactive",
+	    i, &ninactive, size_t);
+
+	emitter_table_printf(emitter,
+	    "HPA shard stats:\n"
+	    "  In full slabs:\n"
+	    "      npageslabs: %zu\n"
+	    "      nactive: %zu\n"
+	    "      ninactive: %zu\n",
+	    npageslabs, nactive, ninactive);
+	emitter_json_object_kv_begin(emitter, "hpa_shard");
+	emitter_json_object_kv_begin(emitter, "full_slabs");
+	emitter_json_kv(emitter, "npageslabs", emitter_type_size, &npageslabs);
+	emitter_json_kv(emitter, "nactive", emitter_type_size, &nactive);
+	emitter_json_kv(emitter, "ninactive", emitter_type_size, &ninactive);
+	emitter_json_object_end(emitter); /* End "full_slabs" */
+
+	COL_HDR(row, size, NULL, right, 20, size)
+	COL_HDR(row, ind, NULL, right, 4, unsigned)
+	COL_HDR(row, npageslabs, NULL, right, 13, size)
+	COL_HDR(row, nactive, NULL, right, 13, size)
+	COL_HDR(row, ninactive, NULL, right, 13, size)
+
+	emitter_table_row(emitter, &header_row);
+	emitter_json_array_kv_begin(emitter, "nonfull_slabs");
+	bool in_gap = false;
+	for (pszind_t j = 0; j < PSSET_NPSIZES; j++) {
+		CTL_M2_M5_GET(
+		    "stats.arenas.0.hpa_shard.nonfull_slabs.0.npageslabs",
+		    i, j, &npageslabs, size_t);
+		CTL_M2_M5_GET(
+		    "stats.arenas.0.hpa_shard.nonfull_slabs.0.nactive",
+		    i, j, &nactive, size_t);
+		CTL_M2_M5_GET(
+		    "stats.arenas.0.hpa_shard.nonfull_slabs.0.ninactive",
+		    i, j, &ninactive, size_t);
+
+		bool in_gap_prev = in_gap;
+		in_gap = (npageslabs == 0);
+		if (in_gap_prev && !in_gap) {
+			emitter_table_printf(emitter,
+			    "                     ---\n");
+		}
+
+		col_size.size_val = sz_pind2sz(j);
+		col_ind.size_val = j;
+		col_npageslabs.size_val = npageslabs;
+		col_nactive.size_val = nactive;
+		col_ninactive.size_val = ninactive;
+		if (!in_gap) {
+			emitter_table_row(emitter, &row);
+		}
+
+		emitter_json_object_begin(emitter);
+		emitter_json_kv(emitter, "npageslabs", emitter_type_size,
+		    &npageslabs);
+		emitter_json_kv(emitter, "nactive", emitter_type_size,
+		    &nactive);
+		emitter_json_kv(emitter, "ninactive", emitter_type_size,
+		    &ninactive);
+		emitter_json_object_end(emitter);
+	}
+	emitter_json_array_end(emitter); /* End "nonfull_slabs" */
+	emitter_json_object_end(emitter); /* End "hpa_shard" */
+}
+
 static void
 stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) {
 	emitter_row_t row;
@@ -677,7 +768,7 @@ stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptim
 
 static void
 stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
-    bool mutex, bool extents) {
+    bool mutex, bool extents, bool hpa) {
 	unsigned nthreads;
 	const char *dss;
 	ssize_t dirty_decay_ms, muzzy_decay_ms;
@@ -997,6 +1088,9 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
 	if (extents) {
 		stats_arena_extents_print(emitter, i);
 	}
+	if (hpa) {
+		stats_arena_hpa_shard_print(emitter, i);
+	}
 }
 
 static void
@@ -1272,7 +1366,7 @@ stats_general_print(emitter_t *emitter) {
 
 static void
 stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
-    bool unmerged, bool bins, bool large, bool mutex, bool extents) {
+    bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) {
 	/*
 	 * These should be deleted.  We keep them around for a while, to aid in
 	 * the transition to the emitter code.
@@ -1405,7 +1499,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 			emitter_table_printf(emitter, "Merged arenas stats:\n");
 			emitter_json_object_kv_begin(emitter, "merged");
 			stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins,
-			    large, mutex, extents);
+			    large, mutex, extents, hpa);
 			emitter_json_object_end(emitter); /* Close "merged". */
 		}
 
@@ -1416,7 +1510,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 			    "Destroyed arenas stats:\n");
 			emitter_json_object_kv_begin(emitter, "destroyed");
 			stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED,
-			    bins, large, mutex, extents);
+			    bins, large, mutex, extents, hpa);
 			emitter_json_object_end(emitter); /* Close "destroyed". */
 		}
 
@@ -1432,7 +1526,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 					emitter_table_printf(emitter,
 					    "arenas[%s]:\n", arena_ind_str);
 					stats_arena_print(emitter, i, bins,
-					    large, mutex, extents);
+					    large, mutex, extents, hpa);
 					/* Close "<arena-ind>". */
 					emitter_json_object_end(emitter);
 				}
@@ -1497,7 +1591,7 @@ stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) {
 	}
 	if (config_stats) {
 		stats_print_helper(&emitter, merged, destroyed, unmerged,
-		    bins, large, mutex, extents);
+		    bins, large, mutex, extents, hpa);
 	}
 
 	emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */