From 392f645f4d850d2256443299183123258899bb3e Mon Sep 17 00:00:00 2001
From: David Goldblatt <davidgoldblatt@fb.com>
Date: Thu, 11 Jun 2020 15:15:51 -0700
Subject: [PATCH] Edata: split up different list linkage uses.

---
 include/jemalloc/internal/arena_structs.h |  2 +-
 include/jemalloc/internal/bin.h           |  2 +-
 include/jemalloc/internal/edata.h         | 37 ++++++++++++++---------
 include/jemalloc/internal/edata_cache.h   |  2 +-
 include/jemalloc/internal/eset.h          |  2 +-
 src/arena.c                               | 14 ++++-----
 src/bin.c                                 |  2 +-
 src/edata_cache.c                         | 10 +++---
 src/eset.c                                |  6 ++--
 src/extent.c                              |  2 +-
 src/large.c                               |  6 ++--
 src/pa.c                                  | 17 ++++++-----
 12 files changed, 55 insertions(+), 47 deletions(-)

diff --git a/include/jemalloc/internal/arena_structs.h b/include/jemalloc/internal/arena_structs.h
index 0c3f42f3..baa7031c 100644
--- a/include/jemalloc/internal/arena_structs.h
+++ b/include/jemalloc/internal/arena_structs.h
@@ -69,7 +69,7 @@ struct arena_s {
 	 *
 	 * Synchronization: large_mtx.
 	 */
-	edata_list_t		large;
+	edata_list_active_t	large;
 	/* Synchronizes all large allocation/update/deallocation. */
 	malloc_mutex_t		large_mtx;
 
diff --git a/include/jemalloc/internal/bin.h b/include/jemalloc/internal/bin.h
index 9a774e90..9241ee73 100644
--- a/include/jemalloc/internal/bin.h
+++ b/include/jemalloc/internal/bin.h
@@ -32,7 +32,7 @@ struct bin_s {
 	edata_heap_t		slabs_nonfull;
 
 	/* List used to track full slabs. */
-	edata_list_t		slabs_full;
+	edata_list_active_t	slabs_full;
 
 	/* Bin statistics. */
 	bin_stats_t	stats;
diff --git a/include/jemalloc/internal/edata.h b/include/jemalloc/internal/edata.h
index 58bddd1a..fb0b489e 100644
--- a/include/jemalloc/internal/edata.h
+++ b/include/jemalloc/internal/edata.h
@@ -185,22 +185,28 @@ struct edata_s {
 		size_t			e_bsize;
 	};
 
-	/*
-	 * List linkage, used by a variety of lists:
-	 * - bin_t's slabs_full
-	 * - extents_t's LRU
-	 * - stashed dirty extents
-	 * - arena's large allocations
-	 */
-	ql_elm(edata_t) ql_link;
-
-	/*
-	 * Linkage for per size class sn/address-ordered heaps, and
-	 * for extent_avail
-	 */
-	phn(edata_t)		ph_link;
+	union {
+		/*
+		 * List linkage used when the edata_t is active; either in
+		 * arena's large allocations or bin_t's slabs_full.
+		 */
+		ql_elm(edata_t)	ql_link_active;
+		/*
+		 * Pairing heap linkage.  Used whenever the extent is inactive
+		 * (in the page allocators), or when it is active and in
+		 * slabs_nonfull, or when the edata_t is unassociated with an
+		 * extent and sitting in an edata_cache.
+		 */
+		phn(edata_t)	ph_link;
+	};
 
 	union {
+		/*
+		 * List linkage used when the extent is inactive:
+		 * - Stashed dirty extents
+		 * - Ecache LRU functionality.
+		 */
+		ql_elm(edata_t) ql_link_inactive;
 		/* Small region slab metadata. */
 		slab_data_t	e_slab_data;
 
@@ -209,7 +215,8 @@ struct edata_s {
 	};
 };
 
-TYPED_LIST(edata_list, edata_t, ql_link)
+TYPED_LIST(edata_list_active, edata_t, ql_link_active)
+TYPED_LIST(edata_list_inactive, edata_t, ql_link_inactive)
 
 static inline unsigned
 edata_arena_ind_get(const edata_t *edata) {
diff --git a/include/jemalloc/internal/edata_cache.h b/include/jemalloc/internal/edata_cache.h
index 620360d1..02685c87 100644
--- a/include/jemalloc/internal/edata_cache.h
+++ b/include/jemalloc/internal/edata_cache.h
@@ -27,7 +27,7 @@ void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache);
 
 typedef struct edata_cache_small_s edata_cache_small_t;
 struct edata_cache_small_s {
-	edata_list_t list;
+	edata_list_inactive_t list;
 	size_t count;
 	edata_cache_t *fallback;
 };
diff --git a/include/jemalloc/internal/eset.h b/include/jemalloc/internal/eset.h
index e29179d1..d260bc13 100644
--- a/include/jemalloc/internal/eset.h
+++ b/include/jemalloc/internal/eset.h
@@ -25,7 +25,7 @@ struct eset_s {
 	bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)];
 
 	/* LRU of all extents in heaps. */
-	edata_list_t lru;
+	edata_list_inactive_t lru;
 
 	/* Page sum for all extents in heaps. */
 	atomic_zu_t npages;
diff --git a/src/arena.c b/src/arena.c
index 2a3af5c6..573dde99 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -594,7 +594,7 @@ arena_bin_slabs_full_insert(arena_t *arena, bin_t *bin, edata_t *slab) {
 	if (arena_is_auto(arena)) {
 		return;
 	}
-	edata_list_append(&bin->slabs_full, slab);
+	edata_list_active_append(&bin->slabs_full, slab);
 }
 
 static void
@@ -602,7 +602,7 @@ arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, edata_t *slab) {
 	if (arena_is_auto(arena)) {
 		return;
 	}
-	edata_list_remove(&bin->slabs_full, slab);
+	edata_list_active_remove(&bin->slabs_full, slab);
 }
 
 static void
@@ -622,8 +622,8 @@ arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) {
 		arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);
 		malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
 	}
-	for (slab = edata_list_first(&bin->slabs_full); slab != NULL;
-	     slab = edata_list_first(&bin->slabs_full)) {
+	for (slab = edata_list_active_first(&bin->slabs_full); slab != NULL;
+	     slab = edata_list_active_first(&bin->slabs_full)) {
 		arena_bin_slabs_full_remove(arena, bin, slab);
 		malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
 		arena_slab_dalloc(tsd_tsdn(tsd), arena, slab);
@@ -655,8 +655,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
 	/* Large allocations. */
 	malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
 
-	for (edata_t *edata = edata_list_first(&arena->large); edata !=
-	    NULL; edata = edata_list_first(&arena->large)) {
+	for (edata_t *edata = edata_list_active_first(&arena->large);
+	    edata != NULL; edata = edata_list_active_first(&arena->large)) {
 		void *ptr = edata_base_get(edata);
 		size_t usize;
 
@@ -1465,7 +1465,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 	atomic_store_u(&arena->dss_prec, (unsigned)extent_dss_prec_get(),
 	    ATOMIC_RELAXED);
 
-	edata_list_init(&arena->large);
+	edata_list_active_init(&arena->large);
 	if (malloc_mutex_init(&arena->large_mtx, "arena_large",
 	    WITNESS_RANK_ARENA_LARGE, malloc_mutex_rank_exclusive)) {
 		goto label_error;
diff --git a/src/bin.c b/src/bin.c
index 52de9ff3..fa204587 100644
--- a/src/bin.c
+++ b/src/bin.c
@@ -46,7 +46,7 @@ bin_init(bin_t *bin) {
 	}
 	bin->slabcur = NULL;
 	edata_heap_new(&bin->slabs_nonfull);
-	edata_list_init(&bin->slabs_full);
+	edata_list_active_init(&bin->slabs_full);
 	if (config_stats) {
 		memset(&bin->stats, 0, sizeof(bin_stats_t));
 	}
diff --git a/src/edata_cache.c b/src/edata_cache.c
index 4601f33c..d899ce80 100644
--- a/src/edata_cache.c
+++ b/src/edata_cache.c
@@ -59,7 +59,7 @@ edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache) {
 
 void
 edata_cache_small_init(edata_cache_small_t *ecs, edata_cache_t *fallback) {
-	edata_list_init(&ecs->list);
+	edata_list_inactive_init(&ecs->list);
 	ecs->count = 0;
 	ecs->fallback = fallback;
 }
@@ -67,9 +67,9 @@ edata_cache_small_init(edata_cache_small_t *ecs, edata_cache_t *fallback) {
 edata_t *
 edata_cache_small_get(edata_cache_small_t *ecs) {
 	assert(ecs->count > 0);
-	edata_t *edata = edata_list_first(&ecs->list);
+	edata_t *edata = edata_list_inactive_first(&ecs->list);
 	assert(edata != NULL);
-	edata_list_remove(&ecs->list, edata);
+	edata_list_inactive_remove(&ecs->list, edata);
 	ecs->count--;
 	return edata;
 }
@@ -77,7 +77,7 @@ edata_cache_small_get(edata_cache_small_t *ecs) {
 void
 edata_cache_small_put(edata_cache_small_t *ecs, edata_t *edata) {
 	assert(edata != NULL);
-	edata_list_append(&ecs->list, edata);
+	edata_list_inactive_append(&ecs->list, edata);
 	ecs->count++;
 }
 
@@ -93,7 +93,7 @@ bool edata_cache_small_prepare(tsdn_t *tsdn, edata_cache_small_t *ecs,
 		if (edata == NULL) {
 			return true;
 		}
-		ql_elm_new(edata, ql_link);
+		ql_elm_new(edata, ql_link_inactive);
 		edata_cache_small_put(ecs, edata);
 	}
 	return false;
diff --git a/src/eset.c b/src/eset.c
index c4e39d25..c9af80e1 100644
--- a/src/eset.c
+++ b/src/eset.c
@@ -12,7 +12,7 @@ eset_init(eset_t *eset, extent_state_t state) {
 		edata_heap_new(&eset->heaps[i]);
 	}
 	bitmap_init(eset->bitmap, &eset_bitmap_info, true);
-	edata_list_init(&eset->lru);
+	edata_list_inactive_init(&eset->lru);
 	atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED);
 	eset->state = state;
 }
@@ -65,7 +65,7 @@ eset_insert(eset_t *eset, edata_t *edata) {
 		eset_stats_add(eset, pind, size);
 	}
 
-	edata_list_append(&eset->lru, edata);
+	edata_list_inactive_append(&eset->lru, edata);
 	size_t npages = size >> LG_PAGE;
 	/*
 	 * All modifications to npages hold the mutex (as asserted above), so we
@@ -95,7 +95,7 @@ eset_remove(eset_t *eset, edata_t *edata) {
 		bitmap_set(eset->bitmap, &eset_bitmap_info,
 		    (size_t)pind);
 	}
-	edata_list_remove(&eset->lru, edata);
+	edata_list_inactive_remove(&eset->lru, edata);
 	size_t npages = size >> LG_PAGE;
 	/*
 	 * As in eset_insert, we hold eset->mtx and so don't need atomic
diff --git a/src/extent.c b/src/extent.c
index 073f8065..d6349c31 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -139,7 +139,7 @@ ecache_evict(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	edata_t *edata;
 	while (true) {
 		/* Get the LRU extent, if any. */
-		edata = edata_list_first(&ecache->eset.lru);
+		edata = edata_list_inactive_first(&ecache->eset.lru);
 		if (edata == NULL) {
 			goto label_return;
 		}
diff --git a/src/large.c b/src/large.c
index 3ea08be4..42d2fd7d 100644
--- a/src/large.c
+++ b/src/large.c
@@ -43,7 +43,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
 	if (!arena_is_auto(arena)) {
 		/* Insert edata into large. */
 		malloc_mutex_lock(tsdn, &arena->large_mtx);
-		edata_list_append(&arena->large, edata);
+		edata_list_active_append(&arena->large, edata);
 		malloc_mutex_unlock(tsdn, &arena->large_mtx);
 	}
 
@@ -225,14 +225,14 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
 		/* See comments in arena_bin_slabs_full_insert(). */
 		if (!arena_is_auto(arena)) {
 			malloc_mutex_lock(tsdn, &arena->large_mtx);
-			edata_list_remove(&arena->large, edata);
+			edata_list_active_remove(&arena->large, edata);
 			malloc_mutex_unlock(tsdn, &arena->large_mtx);
 		}
 	} else {
 		/* Only hold the large_mtx if necessary. */
 		if (!arena_is_auto(arena)) {
 			malloc_mutex_assert_owner(tsdn, &arena->large_mtx);
-			edata_list_remove(&arena->large, edata);
+			edata_list_active_remove(&arena->large, edata);
 		}
 	}
 	arena_extent_dalloc_large_prep(tsdn, arena, edata);
diff --git a/src/pa.c b/src/pa.c
index a7fe70fb..50c64b4f 100644
--- a/src/pa.c
+++ b/src/pa.c
@@ -239,7 +239,8 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
 
 static size_t
 pa_stash_decayed(tsdn_t *tsdn, pa_shard_t *shard, ecache_t *ecache,
-    size_t npages_limit, size_t npages_decay_max, edata_list_t *result) {
+    size_t npages_limit, size_t npages_decay_max,
+    edata_list_inactive_t *result) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
 	    WITNESS_RANK_CORE, 0);
 	ehooks_t *ehooks = pa_shard_ehooks_get(shard);
@@ -252,7 +253,7 @@ pa_stash_decayed(tsdn_t *tsdn, pa_shard_t *shard, ecache_t *ecache,
 		if (edata == NULL) {
 			break;
 		}
-		edata_list_append(result, edata);
+		edata_list_inactive_append(result, edata);
 		nstashed += edata_size_get(edata) >> LG_PAGE;
 	}
 	return nstashed;
@@ -261,7 +262,7 @@ pa_stash_decayed(tsdn_t *tsdn, pa_shard_t *shard, ecache_t *ecache,
 static size_t
 pa_decay_stashed(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
     pa_shard_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
-    edata_list_t *decay_extents) {
+    edata_list_inactive_t *decay_extents) {
 	bool err;
 
 	size_t nmadvise = 0;
@@ -272,9 +273,9 @@ pa_decay_stashed(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
 
 	bool try_muzzy = !fully_decay && pa_shard_may_have_muzzy(shard);
 
-	for (edata_t *edata = edata_list_first(decay_extents); edata !=
-	    NULL; edata = edata_list_first(decay_extents)) {
-		edata_list_remove(decay_extents, edata);
+	for (edata_t *edata = edata_list_inactive_first(decay_extents);
+	    edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
+		edata_list_inactive_remove(decay_extents, edata);
 
 		size_t size = edata_size_get(edata);
 		size_t npages = size >> LG_PAGE;
@@ -342,8 +343,8 @@ pa_decay_to_limit(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
 	decay->purging = true;
 	malloc_mutex_unlock(tsdn, &decay->mtx);
 
-	edata_list_t decay_extents;
-	edata_list_init(&decay_extents);
+	edata_list_inactive_t decay_extents;
+	edata_list_inactive_init(&decay_extents);
 	size_t npurge = pa_stash_decayed(tsdn, shard, ecache, npages_limit,
 	    npages_decay_max, &decay_extents);
 	if (npurge != 0) {