psset: Use fit/insert/remove as basis functions.

All other functionality can be implemented in terms of these; doing so (while
retaining the same API) will be convenient for subsequent refactors.

include/jemalloc/internal/hpdata.h

@@ -123,6 +123,11 @@ TYPED_LIST(hpdata_list, hpdata_t, ql_link)
 typedef ph(hpdata_t) hpdata_age_heap_t;
 ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t);
 
+static inline bool
+hpdata_empty(hpdata_t *hpdata) {
+	return hpdata_nfree_get(hpdata) == HUGEPAGE_PAGES;
+}
+
 void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
 /*
  * Given an hpdata which can serve an allocation request, pick and reserve an

include/jemalloc/internal/psset.h

@@ -64,13 +64,8 @@ void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src);
 void psset_insert(psset_t *psset, hpdata_t *ps);
 void psset_remove(psset_t *psset, hpdata_t *ps);
 
-void psset_hugify(psset_t *psset, hpdata_t *ps);
+/* Analogous to the eset_fit; pick a hpdata to serve the request. */
-
+hpdata_t *psset_fit(psset_t *psset, size_t size);
-/*
- * Tries to obtain a chunk from an existing pageslab already in the set.
- * Returns true on failure.
- */
-bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
 
 /*
  * Given a newly created pageslab ps (not currently in the set), pass ownership
@@ -79,6 +74,7 @@ bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
  */
 void psset_alloc_new(psset_t *psset, hpdata_t *ps,
     edata_t *r_edata, size_t size);
+bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size);
 
 /*
  * Given an extent that comes from a pageslab in this pageslab set, returns it

src/hpa.c

@@ -129,7 +129,7 @@ hpa_hugify(hpdata_t *ps) {
 	assert(hpdata_huge_get(ps));
 	bool err = pages_huge(hpdata_addr_get(ps), HUGEPAGE);
 	/*
-	 * Eat the error; even if the hugeification failed, it's still safe to
+	 * Eat the error; even if the hugification failed, it's still safe to
 	 * pretend it didn't (and would require extraordinary measures to
 	 * unhugify).
 	 */
@@ -233,7 +233,7 @@ hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, hpdata_t *ps) {
 
 	/*
 	 * We do this unconditionally, even for pages which were not originally
-	 * hugeified; it has the same effect.
+	 * hugified; it has the same effect.
 	 */
 	hpa_dehugify(ps);
 
@@ -293,7 +293,9 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
 		 * Do the metadata modification while holding the lock; we'll
 		 * actually change state with the lock dropped.
 		 */
-		psset_hugify(&shard->psset, ps);
+		psset_remove(&shard->psset, ps);
+		hpdata_huge_set(ps, true);
+		psset_insert(&shard->psset, ps);
 	}
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 	if (hugify) {
@@ -463,8 +465,8 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
 	emap_deregister_boundary(tsdn, shard->emap, edata);
 	malloc_mutex_lock(tsdn, &shard->mtx);
 	/*
-	 * Note that the shard mutex protects the edata hugeified field, too.
+	 * Note that the shard mutex protects the edata hugified field, too.
-	 * Page slabs can move between pssets (and have their hugeified status
+	 * Page slabs can move between pssets (and have their hugified status
 	 * change) in racy ways.
 	 */
 	hpdata_t *evicted_ps = psset_dalloc(&shard->psset, edata);

src/psset.c

@@ -76,23 +76,6 @@ psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) {
 	psset_bin_stats_insert_remove(binstats, ps, /* insert */ false);
 }
 
-/*
- * We don't currently need an "activate" equivalent to this, since down the
- * allocation pathways we don't do the optimization in which we change a slab
- * without first removing it from a bin.
- */
-static void
-psset_bin_stats_deactivate(psset_bin_stats_t *binstats, bool huge, size_t num) {
-	size_t *nactive_dst = huge
-	    ? &binstats->nactive_huge : &binstats->nactive_nonhuge;
-	size_t *ninactive_dst = huge
-	    ? &binstats->ninactive_huge : &binstats->ninactive_nonhuge;
-
-	assert(*nactive_dst >= num);
-	*nactive_dst -= num;
-	*ninactive_dst += num;
-}
-
 static void
 psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
 	hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
@@ -148,32 +131,8 @@ psset_remove(psset_t *psset, hpdata_t *ps) {
 	}
 }
 
-void
+hpdata_t *
-psset_hugify(psset_t *psset, hpdata_t *ps) {
+psset_fit(psset_t *psset, size_t size) {
-	assert(!hpdata_huge_get(ps));
-	hpdata_assert_consistent(ps);
-
-	size_t longest_free_range = hpdata_longest_free_range_get(ps);
-	psset_bin_stats_t *bin_stats;
-	if (longest_free_range == 0) {
-		bin_stats = &psset->stats.full_slabs;
-	} else {
-		pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
-		    longest_free_range << LG_PAGE));
-		assert(pind < PSSET_NPSIZES);
-		bin_stats = &psset->stats.nonfull_slabs[pind];
-	}
-	psset_bin_stats_remove(bin_stats, ps);
-	hpdata_huge_set(ps, true);
-	psset_bin_stats_insert(bin_stats, ps);
-}
-
-/*
- * Similar to PAC's extent_recycle_extract.  Out of all the pageslabs in the
- * set, picks one that can satisfy the allocation and remove it from the set.
- */
-static hpdata_t *
-psset_recycle_extract(psset_t *psset, size_t size) {
 	pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
 	pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
 	    (size_t)min_pind);
@@ -185,22 +144,14 @@ psset_recycle_extract(psset_t *psset, size_t size) {
 		return NULL;
 	}
 
-	psset_hpdata_heap_remove(psset, pind, ps);
-	if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
-		bitmap_set(psset->bitmap, &psset_bitmap_info, pind);
-	}
-
 	hpdata_assert_consistent(ps);
+
 	return ps;
 }
 
-/*
+void
- * Given a pageslab ps and an edata to allocate size bytes from, initializes the
+psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
- * edata with a range in the pageslab, and puts ps back in the set.
+	hpdata_assert_empty(ps);
- */
-static void
-psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
-    size_t size) {
 	size_t npages = size / PAGE;
 	size_t begin = hpdata_reserve_alloc(ps, npages);
 	uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
@@ -209,30 +160,28 @@ psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
 	    /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
 	    EXTENT_NOT_HEAD);
 	edata_ps_set(r_edata, ps);
-	/* The pageslab isn't in a bin, so no bin stats need to change. */
-
-	size_t longest_free_range = hpdata_longest_free_range_get(ps);
-	if (longest_free_range == 0) {
-		psset_bin_stats_insert(&psset->stats.full_slabs, ps);
-	} else {
 	psset_insert(psset, ps);
-	}
 }
 
 bool
 psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) {
-	hpdata_t *ps = psset_recycle_extract(psset, size);
+       hpdata_t *ps = psset_fit(psset, size);
        if (ps == NULL) {
                return true;
        }
-	psset_ps_alloc_insert(psset, ps, r_edata, size);
+       psset_remove(psset, ps);
-	return false;
-}
 
-void
+       size_t npages = size / PAGE;
-psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
+       size_t begin = hpdata_reserve_alloc(ps, npages);
-	hpdata_assert_empty(ps);
+       uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
-	psset_ps_alloc_insert(psset, ps, r_edata, size);
+       edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size,
+	   /* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active,
+	   /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
+	   EXTENT_NOT_HEAD);
+       edata_ps_set(r_edata, ps);
+       psset_insert(psset, ps);
+
+       return false;
 }
 
 hpdata_t *
@@ -241,70 +190,17 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
 	assert(edata_ps_get(edata) != NULL);
 	hpdata_t *ps = edata_ps_get(edata);
 
-	size_t ps_old_longest_free_range = hpdata_longest_free_range_get(ps);
-	pszind_t old_pind = SC_NPSIZES;
-	if (ps_old_longest_free_range != 0) {
-		old_pind = sz_psz2ind(sz_psz_quantize_floor(
-		    ps_old_longest_free_range << LG_PAGE));
-	}
-
 	size_t begin =
 	    ((uintptr_t)edata_base_get(edata) - (uintptr_t)hpdata_addr_get(ps))
 	    >> LG_PAGE;
 	size_t len = edata_size_get(edata) >> LG_PAGE;
 
-	/* The pageslab is still in the bin; adjust its stats first. */
+	psset_remove(psset, ps);
-	psset_bin_stats_t *bin_stats = (ps_old_longest_free_range == 0
-	    ? &psset->stats.full_slabs : &psset->stats.nonfull_slabs[old_pind]);
-	psset_bin_stats_deactivate(bin_stats, hpdata_huge_get(ps), len);
-
 	hpdata_unreserve(ps, begin, len);
-	size_t ps_new_longest_free_range = hpdata_longest_free_range_get(ps);
+	if (hpdata_empty(ps)) {
-
-	/*
-	 * If the new free range is no longer than the previous longest one,
-	 * then the pageslab is non-empty and doesn't need to change bins.
-	 * We're done, and don't need to return a pageslab to evict.
-	 */
-	if (ps_new_longest_free_range <= ps_old_longest_free_range) {
-		return NULL;
-	}
-	/*
-	 * If it was previously non-full, then it's in some (possibly now
-	 * incorrect) bin already; remove it.
-	 *
-	 * TODO: We bailed out early above if we didn't expand the longest free
-	 * range, which should avoid a lot of redundant remove/reinserts in the
-	 * same bin.  But it doesn't eliminate all of them; it's possible that
-	 * we decreased the longest free range length, but only slightly, and
-	 * not enough to change our pszind.  We could check that more precisely.
-	 * (Or, ideally, size class dequantization will happen at some point,
-	 * and the issue becomes moot).
-	 */
-	if (ps_old_longest_free_range > 0) {
-		psset_hpdata_heap_remove(psset, old_pind, ps);
-		if (hpdata_age_heap_empty(&psset->pageslabs[old_pind])) {
-			bitmap_set(psset->bitmap, &psset_bitmap_info,
-			    (size_t)old_pind);
-		}
-	} else {
-		/*
-		 * Otherwise, the bin was full, and we need to adjust the full
-		 * bin stats.
-		 */
-		psset_bin_stats_remove(&psset->stats.full_slabs, ps);
-	}
-	/* If the pageslab is empty, it gets evicted from the set. */
-	if (ps_new_longest_free_range == HUGEPAGE_PAGES) {
 		return ps;
-	}
+	} else {
-	/* Otherwise, it gets reinserted. */
+		psset_insert(psset, ps);
-	pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor(
-	    ps_new_longest_free_range << LG_PAGE));
-	if (hpdata_age_heap_empty(&psset->pageslabs[new_pind])) {
-		bitmap_unset(psset->bitmap, &psset_bitmap_info,
-		    (size_t)new_pind);
-	}
-	psset_hpdata_heap_insert(psset, new_pind, ps);
 		return NULL;
+	}
 }