PA->PAC: Move in PAI implementation.

include/jemalloc/internal/pa.h

@@ -66,12 +66,7 @@ struct pa_shard_s {
 	 */
 	atomic_zu_t nactive;
 
-	/*
+	/* Allocates from a PAC. */
-	 * An interface for page allocation from the ecache framework (i.e. a
-	 * cascade of ecache_dirty, ecache_muzzy, ecache_retained).  Right now
-	 * this is the *only* pai, but we'll soon grow another.
-	 */
-	pai_t ecache_pai;
 	pac_t pac;
 
 	/* The source of edata_t objects. */

include/jemalloc/internal/pac.h

@@ -1,6 +1,8 @@
 #ifndef JEMALLOC_INTERNAL_PAC_H
 #define JEMALLOC_INTERNAL_PAC_H
 
+#include "jemalloc/internal/pai.h"
+
 /*
  * Page allocator classic; an implementation of the PAI interface that:
  * - Can be used for arenas with custom extent hooks.
@@ -71,6 +73,11 @@ struct pac_stats_s {
 
 typedef struct pac_s pac_t;
 struct pac_s {
+	/*
+	 * Must be the first member (we convert it to a PAC given only a
+	 * pointer).  The handle to the allocation interface.
+	 */
+	pai_t pai;
 	/*
 	 * Collections of extents that were previously allocated.  These are
 	 * used when allocating extents, in an attempt to re-use address space.

src/pa.c

@@ -1,14 +1,6 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/jemalloc_internal_includes.h"
 
-static edata_t *ecache_pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
-    size_t alignment, bool zero);
-static bool ecache_pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-    size_t old_size, size_t new_size, bool zero);
-static bool ecache_pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-    size_t old_size, size_t new_size);
-static void ecache_pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata);
-
 static void
 pa_nactive_add(pa_shard_t *shard, size_t add_pages) {
 	atomic_fetch_add_zu(&shard->nactive, add_pages, ATOMIC_RELAXED);
@@ -44,11 +36,6 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
 	shard->emap = emap;
 	shard->base = base;
 
-	shard->ecache_pai.alloc = &ecache_pai_alloc;
-	shard->ecache_pai.expand = &ecache_pai_expand;
-	shard->ecache_pai.shrink = &ecache_pai_shrink;
-	shard->ecache_pai.dalloc = &ecache_pai_dalloc;
-
 	return false;
 }
 
@@ -62,43 +49,13 @@ pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
 	pac_destroy(tsdn, &shard->pac);
 }
 
-static inline bool
-pa_shard_may_have_muzzy(pa_shard_t *shard) {
-	return pac_decay_ms_get(&shard->pac, extent_state_muzzy) != 0;
-}
-
-static edata_t *
-ecache_pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
-    bool zero) {
-	pa_shard_t *shard =
-	    (pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai));
-
-	ehooks_t *ehooks = pa_shard_ehooks_get(shard);
-	edata_t *edata = ecache_alloc(tsdn, &shard->pac, ehooks,
-	    &shard->pac.ecache_dirty, NULL, size, alignment, zero);
-
-	if (edata == NULL && pa_shard_may_have_muzzy(shard)) {
-		edata = ecache_alloc(tsdn, &shard->pac, ehooks,
-		    &shard->pac.ecache_muzzy, NULL, size, alignment, zero);
-	}
-	if (edata == NULL) {
-		edata = ecache_alloc_grow(tsdn, &shard->pac, ehooks,
-		    &shard->pac.ecache_retained, NULL, size, alignment, zero);
-		if (config_stats && edata != NULL) {
-			atomic_fetch_add_zu(&shard->pac.stats->pac_mapped, size,
-			    ATOMIC_RELAXED);
-		}
-	}
-	return edata;
-}
-
 edata_t *
 pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
     bool slab, szind_t szind, bool zero) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
 	    WITNESS_RANK_CORE, 0);
 
-	edata_t *edata = pai_alloc(tsdn, &shard->ecache_pai, size, alignment,
+	edata_t *edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment,
 	    zero);
 
 	if (edata != NULL) {
@@ -113,48 +70,6 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
 	return edata;
 }
 
-static bool
-ecache_pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
-    size_t new_size, bool zero) {
-	pa_shard_t *shard =
-	    (pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai));
-
-	ehooks_t *ehooks = pa_shard_ehooks_get(shard);
-	void *trail_begin = edata_past_get(edata);
-
-	size_t mapped_add = 0;
-	size_t expand_amount = new_size - old_size;
-
-	if (ehooks_merge_will_fail(ehooks)) {
-		return true;
-	}
-	edata_t *trail = ecache_alloc(tsdn, &shard->pac, ehooks,
-	    &shard->pac.ecache_dirty, trail_begin, expand_amount, PAGE, zero);
-	if (trail == NULL) {
-		trail = ecache_alloc(tsdn, &shard->pac, ehooks,
-		    &shard->pac.ecache_muzzy, trail_begin, expand_amount, PAGE,
-		    zero);
-	}
-	if (trail == NULL) {
-		trail = ecache_alloc_grow(tsdn, &shard->pac, ehooks,
-		    &shard->pac.ecache_retained, trail_begin, expand_amount,
-		    PAGE, zero);
-		mapped_add = expand_amount;
-	}
-	if (trail == NULL) {
-		return true;
-	}
-	if (extent_merge_wrapper(tsdn, &shard->pac, ehooks, edata, trail)) {
-		extent_dalloc_wrapper(tsdn, &shard->pac, ehooks, trail);
-		return true;
-	}
-	if (config_stats && mapped_add > 0) {
-		atomic_fetch_add_zu(&shard->pac.stats->pac_mapped, mapped_add,
-		    ATOMIC_RELAXED);
-	}
-	return false;
-}
-
 bool
 pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
     size_t new_size, szind_t szind, bool zero) {
@@ -164,7 +79,7 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 
 	size_t expand_amount = new_size - old_size;
 
-	bool error = pai_expand(tsdn, &shard->ecache_pai, edata, old_size,
+	bool error = pai_expand(tsdn, &shard->pac.pai, edata, old_size,
 	    new_size, zero);
 	if (error) {
 		return true;
@@ -176,30 +91,6 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 	return false;
 }
 
-static bool
-ecache_pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
-    size_t new_size) {
-	pa_shard_t *shard =
-	    (pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai));
-
-	ehooks_t *ehooks = pa_shard_ehooks_get(shard);
-	size_t shrink_amount = old_size - new_size;
-
-
-	if (ehooks_split_will_fail(ehooks)) {
-		return true;
-	}
-
-	edata_t *trail = extent_split_wrapper(tsdn, &shard->pac, ehooks, edata,
-	    new_size, shrink_amount);
-	if (trail == NULL) {
-		return true;
-	}
-	ecache_dalloc(tsdn, &shard->pac, ehooks, &shard->pac.ecache_dirty,
-	    trail);
-	return false;
-}
-
 bool
 pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
     size_t new_size, szind_t szind, bool *generated_dirty) {
@@ -209,7 +100,7 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 	size_t shrink_amount = old_size - new_size;
 
 	*generated_dirty = false;
-	bool error = pai_shrink(tsdn, &shard->ecache_pai, edata, old_size,
+	bool error = pai_shrink(tsdn, &shard->pac.pai, edata, old_size,
 	    new_size);
 	if (error) {
 		return true;
@@ -222,15 +113,6 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 	return false;
 }
 
-static void
-ecache_pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
-	pa_shard_t *shard =
-	    (pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai));
-	ehooks_t *ehooks = pa_shard_ehooks_get(shard);
-	ecache_dalloc(tsdn, &shard->pac, ehooks, &shard->pac.ecache_dirty,
-	    edata);
-}
-
 void
 pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
     bool *generated_dirty) {
@@ -241,138 +123,10 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
 	}
 	edata_szind_set(edata, SC_NSIZES);
 	pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE);
-	pai_dalloc(tsdn, &shard->ecache_pai, edata);
+	pai_dalloc(tsdn, &shard->pac.pai, edata);
 	*generated_dirty = true;
 }
 
-static size_t
-pa_stash_decayed(tsdn_t *tsdn, pa_shard_t *shard, ecache_t *ecache,
-    size_t npages_limit, size_t npages_decay_max,
-    edata_list_inactive_t *result) {
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 0);
-	ehooks_t *ehooks = pa_shard_ehooks_get(shard);
-
-	/* Stash extents according to npages_limit. */
-	size_t nstashed = 0;
-	while (nstashed < npages_decay_max) {
-		edata_t *edata = ecache_evict(tsdn, &shard->pac, ehooks, ecache,
-		    npages_limit);
-		if (edata == NULL) {
-			break;
-		}
-		edata_list_inactive_append(result, edata);
-		nstashed += edata_size_get(edata) >> LG_PAGE;
-	}
-	return nstashed;
-}
-
-static size_t
-pa_decay_stashed(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
-    pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
-    edata_list_inactive_t *decay_extents) {
-	bool err;
-
-	size_t nmadvise = 0;
-	size_t nunmapped = 0;
-	size_t npurged = 0;
-
-	ehooks_t *ehooks = pa_shard_ehooks_get(shard);
-
-	bool try_muzzy = !fully_decay && pa_shard_may_have_muzzy(shard);
-
-	for (edata_t *edata = edata_list_inactive_first(decay_extents);
-	    edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
-		edata_list_inactive_remove(decay_extents, edata);
-
-		size_t size = edata_size_get(edata);
-		size_t npages = size >> LG_PAGE;
-
-		nmadvise++;
-		npurged += npages;
-
-		switch (ecache->state) {
-		case extent_state_active:
-			not_reached();
-		case extent_state_dirty:
-			if (try_muzzy) {
-				err = extent_purge_lazy_wrapper(tsdn, ehooks,
-				    edata, /* offset */ 0, size);
-				if (!err) {
-					ecache_dalloc(tsdn, &shard->pac, ehooks,
-					    &shard->pac.ecache_muzzy, edata);
-					break;
-				}
-			}
-			JEMALLOC_FALLTHROUGH;
-		case extent_state_muzzy:
-			extent_dalloc_wrapper(tsdn, &shard->pac, ehooks, edata);
-			nunmapped += npages;
-			break;
-		case extent_state_retained:
-		default:
-			not_reached();
-		}
-	}
-
-	if (config_stats) {
-		LOCKEDINT_MTX_LOCK(tsdn, *shard->stats_mtx);
-		locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
-		    &decay_stats->npurge, 1);
-		locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
-		    &decay_stats->nmadvise, nmadvise);
-		locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
-		    &decay_stats->purged, npurged);
-		LOCKEDINT_MTX_UNLOCK(tsdn, *shard->stats_mtx);
-		atomic_fetch_sub_zu(&shard->pac.stats->pac_mapped,
-		    nunmapped << LG_PAGE, ATOMIC_RELAXED);
-	}
-
-	return npurged;
-}
-
-/*
- * npages_limit: Decay at most npages_decay_max pages without violating the
- * invariant: (ecache_npages_get(ecache) >= npages_limit).  We need an upper
- * bound on number of pages in order to prevent unbounded growth (namely in
- * stashed), otherwise unbounded new pages could be added to extents during the
- * current decay run, so that the purging thread never finishes.
- */
-static void
-pa_decay_to_limit(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
-    pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
-    size_t npages_limit, size_t npages_decay_max) {
-	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
-	    WITNESS_RANK_CORE, 1);
-
-	if (decay->purging || npages_decay_max == 0) {
-		return;
-	}
-	decay->purging = true;
-	malloc_mutex_unlock(tsdn, &decay->mtx);
-
-	edata_list_inactive_t decay_extents;
-	edata_list_inactive_init(&decay_extents);
-	size_t npurge = pa_stash_decayed(tsdn, shard, ecache, npages_limit,
-	    npages_decay_max, &decay_extents);
-	if (npurge != 0) {
-		size_t npurged = pa_decay_stashed(tsdn, shard, decay,
-		    decay_stats, ecache, fully_decay, &decay_extents);
-		assert(npurged == npurge);
-	}
-
-	malloc_mutex_lock(tsdn, &decay->mtx);
-	decay->purging = false;
-}
-
-void
-pa_decay_all(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
-    pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay) {
-	malloc_mutex_assert_owner(tsdn, &decay->mtx);
-	pa_decay_to_limit(tsdn, shard, decay, decay_stats, ecache, fully_decay,
-	    /* npages_limit */ 0, ecache_npages_get(ecache));
-}
-
 bool
 pa_shard_retain_grow_limit_get_set(tsdn_t *tsdn, pa_shard_t *shard,
     size_t *old_limit, size_t *new_limit) {

src/pac.c

@@ -3,6 +3,14 @@
 
 #include "jemalloc/internal/pac.h"
 
+static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size,
+    size_t alignment, bool zero);
+static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+    size_t old_size, size_t new_size, bool zero);
+static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+    size_t old_size, size_t new_size);
+static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata);
+
 static ehooks_t *
 pac_ehooks_get(pac_t *pac) {
 	return base_ehooks_get(pac->base);
@@ -27,8 +35,6 @@ pac_decay_data_get(pac_t *pac, extent_state_t state,
 	}
 }
 
-
-
 bool
 pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
     edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms,
@@ -78,9 +84,113 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	pac->stats = pac_stats;
 	pac->stats_mtx = stats_mtx;
 	atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
+
+	pac->pai.alloc = &pac_alloc_impl;
+	pac->pai.expand = &pac_expand_impl;
+	pac->pai.shrink = &pac_shrink_impl;
+	pac->pai.dalloc = &pac_dalloc_impl;
+
 	return false;
 }
 
+static inline bool
+pac_may_have_muzzy(pac_t *pac) {
+	return pac_decay_ms_get(pac, extent_state_muzzy) != 0;
+}
+
+static edata_t *
+pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
+    bool zero) {
+	pac_t *pac = (pac_t *)self;
+
+	ehooks_t *ehooks = pac_ehooks_get(pac);
+	edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
+	    NULL, size, alignment, zero);
+
+	if (edata == NULL && pac_may_have_muzzy(pac)) {
+		edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
+		    NULL, size, alignment, zero);
+	}
+	if (edata == NULL) {
+		edata = ecache_alloc_grow(tsdn, pac, ehooks,
+		    &pac->ecache_retained, NULL, size, alignment, zero);
+		if (config_stats && edata != NULL) {
+			atomic_fetch_add_zu(&pac->stats->pac_mapped, size,
+			    ATOMIC_RELAXED);
+		}
+	}
+	return edata;
+}
+
+static bool
+pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
+    size_t new_size, bool zero) {
+	pac_t *pac = (pac_t *)self;
+
+	ehooks_t *ehooks = pac_ehooks_get(pac);
+	void *trail_begin = edata_past_get(edata);
+
+	size_t mapped_add = 0;
+	size_t expand_amount = new_size - old_size;
+
+	if (ehooks_merge_will_fail(ehooks)) {
+		return true;
+	}
+	edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
+	    trail_begin, expand_amount, PAGE, zero);
+	if (trail == NULL) {
+		trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
+		    trail_begin, expand_amount, PAGE, zero);
+	}
+	if (trail == NULL) {
+		trail = ecache_alloc_grow(tsdn, pac, ehooks,
+		    &pac->ecache_retained, trail_begin, expand_amount, PAGE,
+		    zero);
+		mapped_add = expand_amount;
+	}
+	if (trail == NULL) {
+		return true;
+	}
+	if (extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) {
+		extent_dalloc_wrapper(tsdn, pac, ehooks, trail);
+		return true;
+	}
+	if (config_stats && mapped_add > 0) {
+		atomic_fetch_add_zu(&pac->stats->pac_mapped, mapped_add,
+		    ATOMIC_RELAXED);
+	}
+	return false;
+}
+
+static bool
+pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
+    size_t new_size) {
+	pac_t *pac = (pac_t *)self;
+
+	ehooks_t *ehooks = pac_ehooks_get(pac);
+	size_t shrink_amount = old_size - new_size;
+
+
+	if (ehooks_split_will_fail(ehooks)) {
+		return true;
+	}
+
+	edata_t *trail = extent_split_wrapper(tsdn, pac, ehooks, edata,
+	    new_size, shrink_amount);
+	if (trail == NULL) {
+		return true;
+	}
+	ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail);
+	return false;
+}
+
+static void
+pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
+	pac_t *pac = (pac_t *)self;
+	ehooks_t *ehooks = pac_ehooks_get(pac);
+	ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata);
+}
+
 bool
 pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
     size_t *new_limit) {
@@ -107,7 +217,8 @@ pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
 
 static size_t
 pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
-    size_t npages_limit, size_t npages_decay_max, edata_list_t *result) {
+    size_t npages_limit, size_t npages_decay_max,
+    edata_list_inactive_t *result) {
 	witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
 	    WITNESS_RANK_CORE, 0);
 	ehooks_t *ehooks = pac_ehooks_get(pac);
@@ -120,7 +231,7 @@ pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
 		if (edata == NULL) {
 			break;
 		}
-		edata_list_append(result, edata);
+		edata_list_inactive_append(result, edata);
 		nstashed += edata_size_get(edata) >> LG_PAGE;
 	}
 	return nstashed;
@@ -129,7 +240,7 @@ pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
 static size_t
 pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
     pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
-    edata_list_t *decay_extents) {
+    edata_list_inactive_t *decay_extents) {
 	bool err;
 
 	size_t nmadvise = 0;
@@ -141,9 +252,9 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
 	bool try_muzzy = !fully_decay
 	    && pac_decay_ms_get(pac, extent_state_muzzy) != 0;
 
-	for (edata_t *edata = edata_list_first(decay_extents); edata !=
+	for (edata_t *edata = edata_list_inactive_first(decay_extents); edata !=
-	    NULL; edata = edata_list_first(decay_extents)) {
+	    NULL; edata = edata_list_inactive_first(decay_extents)) {
-		edata_list_remove(decay_extents, edata);
+		edata_list_inactive_remove(decay_extents, edata);
 
 		size_t size = edata_size_get(edata);
 		size_t npages = size >> LG_PAGE;
@@ -211,8 +322,8 @@ pac_decay_to_limit(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
 	decay->purging = true;
 	malloc_mutex_unlock(tsdn, &decay->mtx);
 
-	edata_list_t decay_extents;
+	edata_list_inactive_t decay_extents;
-	edata_list_init(&decay_extents);
+	edata_list_inactive_init(&decay_extents);
 	size_t npurge = pac_stash_decayed(tsdn, pac, ecache, npages_limit,
 	    npages_decay_max, &decay_extents);
 	if (npurge != 0) {