From 722652222a159c10f616d61b6dc145d07f84e025 Mon Sep 17 00:00:00 2001
From: David Goldblatt <davidgoldblatt@fb.com>
Date: Fri, 29 May 2020 17:14:16 -0700
Subject: [PATCH] PAC: Move in edata_cache accesses.

---
 include/jemalloc/internal/pac.h |  2 ++
 src/extent.c                    | 26 +++++++++++++-------------
 src/pa.c                        |  1 +
 src/pa_extra.c                  |  1 -
 4 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/include/jemalloc/internal/pac.h b/include/jemalloc/internal/pac.h
index 73e672f6..bd1c8566 100644
--- a/include/jemalloc/internal/pac.h
+++ b/include/jemalloc/internal/pac.h
@@ -20,6 +20,8 @@ struct pac_s {
 	ecache_t ecache_dirty;
 	ecache_t ecache_muzzy;
 	ecache_t ecache_retained;
+
+	edata_cache_t *edata_cache;
 };
 
 #endif /* JEMALLOC_INTERNAL_PAC_H */
diff --git a/src/extent.c b/src/extent.c
index 3d827b8c..4810a614 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -210,7 +210,7 @@ extents_abandon_vm(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 			    edata_size_get(edata), growing_retained);
 		}
 	}
-	edata_cache_put(tsdn, &shard->edata_cache, edata);
+	edata_cache_put(tsdn, shard->pac.edata_cache, edata);
 }
 
 static void
@@ -632,7 +632,7 @@ extent_grow_retained(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 		    shard->ecache_grow.next + egn_skip);
 	}
 
-	edata_t *edata = edata_cache_get(tsdn, &shard->edata_cache);
+	edata_t *edata = edata_cache_get(tsdn, shard->pac.edata_cache);
 	if (edata == NULL) {
 		goto label_err;
 	}
@@ -643,7 +643,7 @@ extent_grow_retained(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	    &committed);
 
 	if (ptr == NULL) {
-		edata_cache_put(tsdn, &shard->edata_cache, edata);
+		edata_cache_put(tsdn, shard->pac.edata_cache, edata);
 		goto label_err;
 	}
 
@@ -653,7 +653,7 @@ extent_grow_retained(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	    EXTENT_IS_HEAD);
 
 	if (extent_register_no_gdump_add(tsdn, shard, edata)) {
-		edata_cache_put(tsdn, &shard->edata_cache, edata);
+		edata_cache_put(tsdn, shard->pac.edata_cache, edata);
 		goto label_err;
 	}
 
@@ -781,7 +781,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
 	    WITNESS_RANK_CORE, 0);
 
-	edata_t *edata = edata_cache_get(tsdn, &shard->edata_cache);
+	edata_t *edata = edata_cache_get(tsdn, shard->pac.edata_cache);
 	if (edata == NULL) {
 		return NULL;
 	}
@@ -789,7 +789,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	void *addr = ehooks_alloc(tsdn, ehooks, new_addr, size, palignment,
 	    &zero, commit);
 	if (addr == NULL) {
-		edata_cache_put(tsdn, &shard->edata_cache, edata);
+		edata_cache_put(tsdn, shard->pac.edata_cache, edata);
 		return NULL;
 	}
 	edata_init(edata, ecache_ind_get(&shard->pac.ecache_dirty), addr,
@@ -797,7 +797,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	    extent_state_active, zero, *commit, /* ranged */ false,
 	    EXTENT_NOT_HEAD);
 	if (extent_register(tsdn, shard, edata)) {
-		edata_cache_put(tsdn, &shard->edata_cache, edata);
+		edata_cache_put(tsdn, shard->pac.edata_cache, edata);
 		return NULL;
 	}
 
@@ -1000,7 +1000,7 @@ extent_dalloc_gap(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	    WITNESS_RANK_CORE, 0);
 
 	if (extent_register(tsdn, shard, edata)) {
-		edata_cache_put(tsdn, &shard->edata_cache, edata);
+		edata_cache_put(tsdn, shard->pac.edata_cache, edata);
 		return;
 	}
 	extent_dalloc_wrapper(tsdn, shard, ehooks, edata);
@@ -1023,7 +1023,7 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	    edata_size_get(edata), edata_committed_get(edata));
 
 	if (!err) {
-		edata_cache_put(tsdn, &shard->edata_cache, edata);
+		edata_cache_put(tsdn, shard->pac.edata_cache, edata);
 	}
 
 	return err;
@@ -1093,7 +1093,7 @@ extent_destroy_wrapper(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 	ehooks_destroy(tsdn, ehooks, edata_base_get(edata),
 	    edata_size_get(edata), edata_committed_get(edata));
 
-	edata_cache_put(tsdn, &shard->edata_cache, edata);
+	edata_cache_put(tsdn, shard->pac.edata_cache, edata);
 }
 
 static bool
@@ -1177,7 +1177,7 @@ extent_split_impl(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 		return NULL;
 	}
 
-	edata_t *trail = edata_cache_get(tsdn, &shard->edata_cache);
+	edata_t *trail = edata_cache_get(tsdn, shard->pac.edata_cache);
 	if (trail == NULL) {
 		goto label_error_a;
 	}
@@ -1214,7 +1214,7 @@ extent_split_impl(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
 label_error_c:
 	emap_unlock_edata2(tsdn, shard->emap, edata, trail);
 label_error_b:
-	edata_cache_put(tsdn, &shard->edata_cache, trail);
+	edata_cache_put(tsdn, shard->pac.edata_cache, trail);
 label_error_a:
 	return NULL;
 }
@@ -1262,7 +1262,7 @@ extent_merge_impl(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks, edata_t *a,
 	emap_merge_commit(tsdn, shard->emap, &prepare, a, b);
 	emap_unlock_edata2(tsdn, shard->emap, a, b);
 
-	edata_cache_put(tsdn, &shard->edata_cache, b);
+	edata_cache_put(tsdn, shard->pac.edata_cache, b);
 
 	return false;
 }
diff --git a/src/pa.c b/src/pa.c
index 27fc9ee2..f37337de 100644
--- a/src/pa.c
+++ b/src/pa.c
@@ -57,6 +57,7 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
 	if (edata_cache_init(&shard->edata_cache, base)) {
 		return true;
 	}
+	shard->pac.edata_cache = &shard->edata_cache;
 
 	if (ecache_grow_init(tsdn, &shard->ecache_grow)) {
 		return true;
diff --git a/src/pa_extra.c b/src/pa_extra.c
index 70ef19b6..caa94d82 100644
--- a/src/pa_extra.c
+++ b/src/pa_extra.c
@@ -26,7 +26,6 @@ pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_prefork(tsdn, &shard->pac.ecache_retained);
 }
 
-
 void
 pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) {
 	edata_cache_prefork(tsdn, &shard->edata_cache);