Ehooks: Add head tracking.

include/jemalloc/internal/edata.h

@@ -467,20 +467,12 @@ edata_prof_alloc_time_set(edata_t *edata, nstime_t *t) {
 
 static inline bool
 edata_is_head_get(edata_t *edata) {
-	if (maps_coalesce) {
-		not_reached();
-	}
-
 	return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK) >>
 	    EDATA_BITS_IS_HEAD_SHIFT);
 }
 
 static inline void
 edata_is_head_set(edata_t *edata, bool is_head) {
-	if (maps_coalesce) {
-		not_reached();
-	}
-
 	edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK) |
 	    ((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT);
 }
@@ -502,9 +494,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
 	edata_committed_set(edata, committed);
 	edata_dumpable_set(edata, dumpable);
 	ql_elm_new(edata, ql_link);
-	if (!maps_coalesce) {
+	edata_is_head_set(edata, is_head == EXTENT_IS_HEAD);
-		edata_is_head_set(edata, is_head == EXTENT_IS_HEAD);
-	}
 	if (config_prof) {
 		edata_prof_tctx_set(edata, NULL);
 	}

include/jemalloc/internal/ehooks.h

@@ -1,16 +1,21 @@
 #ifndef JEMALLOC_INTERNAL_EHOOKS_H
 #define JEMALLOC_INTERNAL_EHOOKS_H
 
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/extent_mmap.h"
+
 /*
  * This module is the internal interface to the extent hooks (both
  * user-specified and external).  Eventually, this will give us the flexibility
  * to use multiple different versions of user-visible extent-hook APIs under a
  * single user interface.
+ *
+ * Current API expansions (not available to anyone but the default hooks yet):
+ *   - Head state tracking.  Hooks can decide whether or not to merge two
+ *     extents based on whether or not one of them is the head (i.e. was
+ *     allocated on its own).  The later extent loses its "head" status.
  */
 
-#include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/extent_mmap.h"
-
 extern const extent_hooks_t ehooks_default_extent_hooks;
 
 typedef struct ehooks_s ehooks_t;
@@ -43,7 +48,8 @@ bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length);
 bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
 #endif
 bool ehooks_default_split_impl();
-bool ehooks_default_merge_impl(void *addr_a, void *addr_b);
+bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, bool head_a,
+    void *addr_b, bool head_b);
 void ehooks_default_zero_impl(void *addr, size_t size);
 
 /*
@@ -314,10 +320,12 @@ ehooks_split(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
 
 static inline bool
 ehooks_merge(tsdn_t *tsdn, ehooks_t *ehooks, void *addr_a, size_t size_a,
-    void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
+    bool head_a, void *addr_b, size_t size_b, bool head_b, bool committed,
+    unsigned arena_ind) {
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
 	if (extent_hooks == &ehooks_default_extent_hooks) {
-		return ehooks_default_merge_impl(addr_a, addr_b);
+		return ehooks_default_merge_impl(tsdn, addr_a, head_a, addr_b,
+		    head_b);
 	} else if (extent_hooks->merge == NULL) {
 		return true;
 	} else {

include/jemalloc/internal/extent2.h

@@ -54,7 +54,6 @@ edata_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
     size_t size_b, szind_t szind_b, bool slab_b);
 bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
     edata_t *a, edata_t *b);
-bool extent_head_no_merge(edata_t *a, edata_t *b);
 
 bool extent_boot(void);
 

src/ehooks.c

@@ -183,8 +183,51 @@ ehooks_default_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
 	return ehooks_default_split_impl();
 }
 
+static inline bool
+ehooks_same_sn(tsdn_t *tsdn, void *addr_a, void *addr_b) {
+	edata_t *a = iealloc(tsdn, addr_a);
+	edata_t *b = iealloc(tsdn, addr_b);
+	return edata_sn_comp(a, b) == 0;
+}
+
+/*
+ * Returns true if the given extents can't be merged because of their head bit
+ * settings.  Assumes the second extent has the higher address.
+ */
+static bool
+ehooks_no_merge_heads(tsdn_t *tsdn, void *addr_a, bool head_a, void *addr_b,
+    bool head_b) {
+	/*
+	 * When coalesce is not always allowed (Windows), only merge extents
+	 * from the same VirtualAlloc region under opt.retain (in which case
+	 * MEM_DECOMMIT is utilized for purging).
+	 */
+	if (maps_coalesce) {
+		return false;
+	}
+	if (!opt_retain) {
+		return true;
+	}
+	/* If b is a head extent, disallow the cross-region merge. */
+	if (head_b) {
+		/*
+		 * Additionally, sn should not overflow with retain; sanity
+		 * check that different regions have unique sn.
+		 */
+		assert(!ehooks_same_sn(tsdn, addr_a, addr_b));
+		return true;
+	}
+	assert(ehooks_same_sn(tsdn, addr_a, addr_b));
+
+	return false;
+}
+
 bool
-ehooks_default_merge_impl(void *addr_a, void *addr_b) {
+ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, bool head_a, void *addr_b,
+    bool head_b) {
+	if (ehooks_no_merge_heads(tsdn, addr_a, head_a, addr_b, head_b)) {
+		return true;
+	}
 	if (!maps_coalesce && !opt_retain) {
 		return true;
 	}
@@ -198,15 +241,12 @@ ehooks_default_merge_impl(void *addr_a, void *addr_b) {
 static bool
 ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
     void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
-	if (!maps_coalesce) {
+	tsdn_t *tsdn = tsdn_fetch();
-		tsdn_t *tsdn = tsdn_fetch();
+	edata_t *a = iealloc(tsdn, addr_a);
-		edata_t *a = iealloc(tsdn, addr_a);
+	bool head_a = edata_is_head_get(a);
-		edata_t *b = iealloc(tsdn, addr_b);
+	edata_t *b = iealloc(tsdn, addr_b);
-		if (extent_head_no_merge(a, b)) {
+	bool head_b = edata_is_head_get(b);
-			return true;
+	return ehooks_default_merge_impl(tsdn, addr_a, head_a, addr_b, head_b);
-		}
-	}
-	return ehooks_default_merge_impl(addr_a, addr_b);
 }
 
 void

src/extent2.c

@@ -1493,38 +1493,6 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 	    slab_a, size_b, szind_b, slab_b, false);
 }
 
-/*
- * Returns true if the given extents can't be merged because of their head bit
- * settings.  Assumes the second extent has the higher address.
- */
-bool
-extent_head_no_merge(edata_t *a, edata_t *b) {
-	assert(edata_base_get(a) < edata_base_get(b));
-	/*
-	 * When coalesce is not always allowed (Windows), only merge extents
-	 * from the same VirtualAlloc region under opt.retain (in which case
-	 * MEM_DECOMMIT is utilized for purging).
-	 */
-	if (maps_coalesce) {
-		return false;
-	}
-	if (!opt_retain) {
-		return true;
-	}
-	/* If b is a head extent, disallow the cross-region merge. */
-	if (edata_is_head_get(b)) {
-		/*
-		 * Additionally, sn should not overflow with retain; sanity
-		 * check that different regions have unique sn.
-		 */
-		assert(edata_sn_comp(a, b) != 0);
-		return true;
-	}
-	assert(edata_sn_comp(a, b) == 0);
-
-	return false;
-}
-
 static bool
 extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a,
     edata_t *b, bool growing_retained) {
@@ -1532,13 +1500,10 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a,
 	    WITNESS_RANK_CORE, growing_retained ? 1 : 0);
 	assert(edata_base_get(a) < edata_base_get(b));
 
-	if (ehooks_merge_will_fail(ehooks) || extent_head_no_merge(a, b)) {
-		return true;
-	}
-
 	bool err = ehooks_merge(tsdn, ehooks, edata_base_get(a),
-	    edata_size_get(a), edata_base_get(b), edata_size_get(b),
+	    edata_size_get(a), edata_is_head_get(a), edata_base_get(b),
-	    edata_committed_get(a), arena_ind_get(arena));
+	    edata_size_get(b), edata_is_head_get(b), edata_committed_get(a),
+	    arena_ind_get(arena));
 
 	if (err) {
 		return true;