Pairing heap: inline functions instead of macros.

By force-inlining everything that would otherwise be a macro, we get the same
effect (it's not clear in the first place that this is actually a good idea, but
it avoids making any changes to the existing performance profile).

This makes the code more maintainable (in anticipation of subsequent changes),
as well as making performance profiles and debug info more readable (we get
"real" line numbers, instead of making everything point to the macro definition
of all associated functions).

include/jemalloc/internal/edata.h

@@ -81,8 +81,8 @@ struct edata_map_info_s {
 
 /* Extent (span of pages).  Use accessor functions for e_* fields. */
 typedef struct edata_s edata_t;
-typedef ph(edata_t) edata_avail_t;
+ph_structs(edata_avail, edata_t);
-typedef ph(edata_t) edata_heap_t;
+ph_structs(edata_heap, edata_t);
 struct edata_s {
 	/*
 	 * Bitfield containing several fields:
@@ -214,7 +214,10 @@ struct edata_s {
 		 * slabs_nonfull, or when the edata_t is unassociated with an
 		 * extent and sitting in an edata_cache.
 		 */
-		phn(edata_t)	ph_link;
+		union {
+			edata_heap_link_t heap_link;
+			edata_avail_link_t avail_link;
+		};
 	};
 
 	union {
@@ -664,7 +667,7 @@ edata_esnead_comp(const edata_t *a, const edata_t *b) {
 	return ret;
 }
 
-ph_proto(, edata_avail_, edata_avail_t, edata_t)
+ph_proto(, edata_avail, edata_t)
-ph_proto(, edata_heap_, edata_heap_t, edata_t)
+ph_proto(, edata_heap, edata_t)
 
 #endif /* JEMALLOC_INTERNAL_EDATA_H */

include/jemalloc/internal/hpdata.h

@@ -18,6 +18,7 @@
  * hugepage-sized and hugepage-aligned; it's *potentially* huge.
  */
 typedef struct hpdata_s hpdata_t;
+ph_structs(hpdata_age_heap, hpdata_t);
 struct hpdata_s {
 	/*
 	 * We likewise follow the edata convention of mangling names and forcing
@@ -82,7 +83,7 @@ struct hpdata_s {
 
 	union {
 		/* When nonempty (and also nonfull), used by the psset bins. */
-		phn(hpdata_t) ph_link;
+		hpdata_age_heap_link_t age_link;
 		/*
 		 * When empty (or not corresponding to any hugepage), list
 		 * linkage.
@@ -120,8 +121,7 @@ TYPED_LIST(hpdata_empty_list, hpdata_t, ql_link_empty)
 TYPED_LIST(hpdata_purge_list, hpdata_t, ql_link_purge)
 TYPED_LIST(hpdata_hugify_list, hpdata_t, ql_link_hugify)
 
-typedef ph(hpdata_t) hpdata_age_heap_t;
+ph_proto(, hpdata_age_heap, hpdata_t);
-ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t);
 
 static inline void *
 hpdata_addr_get(const hpdata_t *hpdata) {

include/jemalloc/internal/ph.h

@@ -15,377 +15,435 @@
  *******************************************************************************
  */
 
+typedef int (*ph_cmp_t)(void *, void *);
+
 /* Node structure. */
-#define phn(a_type)							\
+typedef struct phn_link_s phn_link_t;
-struct {								\
+struct phn_link_s {
-	a_type	*phn_prev;						\
+	void *prev;
-	a_type	*phn_next;						\
+	void *next;
-	a_type	*phn_lchild;						\
+	void *lchild;
+};
+
+typedef struct ph_s ph_t;
+struct ph_s {
+	void *root;
+};
+
+JEMALLOC_ALWAYS_INLINE phn_link_t *
+phn_link_get(void *phn, size_t offset) {
+	return (phn_link_t *)(((uintptr_t)phn) + offset);
 }
 
-/* Root structure. */
+JEMALLOC_ALWAYS_INLINE void
-#define ph(a_type)							\
+phn_link_init(void *phn, size_t offset) {
-struct {								\
+	phn_link_get(phn, offset)->prev = NULL;
-	a_type	*ph_root;						\
+	phn_link_get(phn, offset)->next = NULL;
+	phn_link_get(phn, offset)->lchild = NULL;
 }
 
-/* Internal utility macros. */
+/* Internal utility helpers. */
-#define phn_lchild_get(a_type, a_field, a_phn)				\
+JEMALLOC_ALWAYS_INLINE void *
-	(a_phn->a_field.phn_lchild)
+phn_lchild_get(void *phn, size_t offset) {
-#define phn_lchild_set(a_type, a_field, a_phn, a_lchild) do {		\
+	return phn_link_get(phn, offset)->lchild;
-	a_phn->a_field.phn_lchild = a_lchild;				\
+}
-} while (0)
 
-#define phn_next_get(a_type, a_field, a_phn)				\
+JEMALLOC_ALWAYS_INLINE void
-	(a_phn->a_field.phn_next)
+phn_lchild_set(void *phn, void *lchild, size_t offset) {
-#define phn_prev_set(a_type, a_field, a_phn, a_prev) do {		\
+	phn_link_get(phn, offset)->lchild = lchild;
-	a_phn->a_field.phn_prev = a_prev;				\
+}
-} while (0)
 
-#define phn_prev_get(a_type, a_field, a_phn)				\
+JEMALLOC_ALWAYS_INLINE void *
-	(a_phn->a_field.phn_prev)
+phn_next_get(void *phn, size_t offset) {
-#define phn_next_set(a_type, a_field, a_phn, a_next) do {		\
+	return phn_link_get(phn, offset)->next;
-	a_phn->a_field.phn_next = a_next;				\
+}
-} while (0)
 
-#define phn_merge_ordered(a_type, a_field, a_phn0, a_phn1, a_cmp) do {	\
+JEMALLOC_ALWAYS_INLINE void
-	a_type *phn0child;						\
+phn_next_set(void *phn, void *next, size_t offset) {
+	phn_link_get(phn, offset)->next = next;
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+phn_prev_get(void *phn, size_t offset) {
+	return phn_link_get(phn, offset)->prev;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+phn_prev_set(void *phn, void *prev, size_t offset) {
+	phn_link_get(phn, offset)->prev = prev;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+phn_merge_ordered(void *phn0, void *phn1, size_t offset,
+    ph_cmp_t cmp) {
+	void *phn0child;
+
+	assert(phn0 != NULL);
+	assert(phn1 != NULL);
+	assert(cmp(phn0, phn1) <= 0);
+
+	phn_prev_set(phn1, phn0, offset);
+	phn0child = phn_lchild_get(phn0, offset);
+	phn_next_set(phn1, phn0child, offset);
+	if (phn0child != NULL) {
+		phn_prev_set(phn0child, phn1, offset);
+	}
+	phn_lchild_set(phn0, phn1, offset);
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+phn_merge(void *phn0, void *phn1, size_t offset, ph_cmp_t cmp) {
+	void *result;
+	if (phn0 == NULL) {
+		result = phn1;
+	} else if (phn1 == NULL) {
+		result = phn0;
+	} else if (cmp(phn0, phn1) < 0) {
+		phn_merge_ordered(phn0, phn1, offset, cmp);
+		result = phn0;
+	} else {
+		phn_merge_ordered(phn1, phn0, offset, cmp);
+		result = phn1;
+	}
+	return result;
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+phn_merge_siblings(void *phn, size_t offset, ph_cmp_t cmp) {
+	void *head = NULL;
+	void *tail = NULL;
+	void *phn0 = phn;
+	void *phn1 = phn_next_get(phn0, offset);
+
+	/*
+	 * Multipass merge, wherein the first two elements of a FIFO
+	 * are repeatedly merged, and each result is appended to the
+	 * singly linked FIFO, until the FIFO contains only a single
+	 * element.  We start with a sibling list but no reference to
+	 * its tail, so we do a single pass over the sibling list to
+	 * populate the FIFO.
+	 */
+	if (phn1 != NULL) {
+		void *phnrest = phn_next_get(phn1, offset);
+		if (phnrest != NULL) {
+			phn_prev_set(phnrest, NULL, offset);
+		}
+		phn_prev_set(phn0, NULL, offset);
+		phn_next_set(phn0, NULL, offset);
+		phn_prev_set(phn1, NULL, offset);
+		phn_next_set(phn1, NULL, offset);
+		phn0 = phn_merge(phn0, phn1, offset, cmp);
+		head = tail = phn0;
+		phn0 = phnrest;
+		while (phn0 != NULL) {
+			phn1 = phn_next_get(phn0, offset);
+			if (phn1 != NULL) {
+				phnrest = phn_next_get(phn1, offset);
+				if (phnrest != NULL) {
+					phn_prev_set(phnrest, NULL, offset);
+				}
+				phn_prev_set(phn0, NULL, offset);
+				phn_next_set(phn0, NULL, offset);
+				phn_prev_set(phn1, NULL, offset);
+				phn_next_set(phn1, NULL, offset);
+				phn0 = phn_merge(phn0, phn1, offset, cmp);
+				phn_next_set(tail, phn0, offset);
+				tail = phn0;
+				phn0 = phnrest;
+			} else {
+				phn_next_set(tail, phn0, offset);
+				tail = phn0;
+				phn0 = NULL;
+			}
+		}
+		phn0 = head;
+		phn1 = phn_next_get(phn0, offset);
+		if (phn1 != NULL) {
+			while (true) {
+				head = phn_next_get(phn1, offset);
+				assert(phn_prev_get(phn0, offset) == NULL);
+				phn_next_set(phn0, NULL, offset);
+				assert(phn_prev_get(phn1, offset) == NULL);
+				phn_next_set(phn1, NULL, offset);
+				phn0 = phn_merge(phn0, phn1, offset, cmp);
+				if (head == NULL) {
+					break;
+				}
+				phn_next_set(tail, phn0, offset);
+				tail = phn0;
+				phn0 = head;
+				phn1 = phn_next_get(phn0, offset);
+			}
+		}
+	}
+	return phn0;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+ph_merge_aux(ph_t *ph, size_t offset, ph_cmp_t cmp) {
+	void *phn = phn_next_get(ph->root, offset);
+	if (phn != NULL) {
+		phn_prev_set(ph->root, NULL, offset);
+		phn_next_set(ph->root, NULL, offset);
+		phn_prev_set(phn, NULL, offset);
+		phn = phn_merge_siblings(phn, offset, cmp);
+		assert(phn_next_get(phn, offset) == NULL);
+		ph->root = phn_merge(ph->root, phn, offset, cmp);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+ph_merge_children(void *phn, size_t offset, ph_cmp_t cmp) {
+	void *result;
+	void *lchild = phn_lchild_get(phn, offset);
+	if (lchild == NULL) {
+		result = NULL;
+	} else {
+		result = phn_merge_siblings(lchild, offset, cmp);
+	}
+	return result;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+ph_new(ph_t *ph) {
+	ph->root = NULL;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+ph_empty(ph_t *ph) {
+	return ph->root == NULL;
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+ph_first(ph_t *ph, size_t offset, ph_cmp_t cmp) {
+	if (ph->root == NULL) {
+		return NULL;
+	}
+	ph_merge_aux(ph, offset, cmp);
+	return ph->root;
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+ph_any(ph_t *ph, size_t offset) {
+	if (ph->root == NULL) {
+		return NULL;
+	}
+	void *aux = phn_next_get(ph->root, offset);
+	if (aux != NULL) {
+		return aux;
+	}
+	return ph->root;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+ph_insert(ph_t *ph, void *phn, size_t offset) {
+	phn_link_init(phn, offset);
+
+	/*
+	 * Treat the root as an aux list during insertion, and lazily merge
+	 * during a_prefix##remove_first().  For elements that are inserted,
+	 * then removed via a_prefix##remove() before the aux list is ever
+	 * processed, this makes insert/remove constant-time, whereas eager
+	 * merging would make insert O(log n).
+	 */
+	if (ph->root == NULL) {
+		ph->root = phn;
+	} else {
+		phn_next_set(phn, phn_next_get(ph->root, offset), offset);
+		if (phn_next_get(ph->root, offset) != NULL) {
+			phn_prev_set(phn_next_get(ph->root, offset), phn,
+			    offset);
+		}
+		phn_prev_set(phn, ph->root, offset);
+		phn_next_set(ph->root, phn, offset);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+ph_remove_first(ph_t *ph, size_t offset, ph_cmp_t cmp) {
+	void *ret;
+
+	if (ph->root == NULL) {
+		return NULL;
+	}
+	ph_merge_aux(ph, offset, cmp);
+	ret = ph->root;
+	ph->root = ph_merge_children(ph->root, offset, cmp);
+
+	return ret;
+
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+ph_remove_any(ph_t *ph, size_t offset, ph_cmp_t cmp) {
+	/*
+	 * Remove the most recently inserted aux list element, or the root if
+	 * the aux list is empty.  This has the effect of behaving as a LIFO
+	 * (and insertion/removal is therefore constant-time) if
+	 * a_prefix##[remove_]first() are never called.
+	 */
+	if (ph->root == NULL) {
+		return NULL;
+	}
+	void *ret = phn_next_get(ph->root, offset);
+	if (ret != NULL) {
+		void *aux = phn_next_get(ret, offset);
+		phn_next_set(ph->root, aux, offset);
+		if (aux != NULL) {
+			phn_prev_set(aux, ph->root, offset);
+		}
+		return ret;
+	}
+	ret = ph->root;
+	ph->root = ph_merge_children(ph->root, offset, cmp);
+	return ret;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+ph_remove(ph_t *ph, void *phn, size_t offset, ph_cmp_t cmp) {
+	void *replace;
+	void *parent;
+
+	if (ph->root == phn) {
+		/*
+		 * We can delete from aux list without merging it, but we need
+		 * to merge if we are dealing with the root node and it has
+		 * children.
+		 */
+		if (phn_lchild_get(phn, offset) == NULL) {
+			ph->root = phn_next_get(phn, offset);
+			if (ph->root != NULL) {
+				phn_prev_set(ph->root, NULL, offset);
+			}
+			return;
+		}
+		ph_merge_aux(ph, offset, cmp);
+		if (ph->root == phn) {
+			ph->root = ph_merge_children(ph->root, offset, cmp);
+			return;
+		}
+	}
+
+	/* Get parent (if phn is leftmost child) before mutating. */
+	if ((parent = phn_prev_get(phn, offset)) != NULL) {
+		if (phn_lchild_get(parent, offset) != phn) {
+			parent = NULL;
+		}
+	}
+	/* Find a possible replacement node, and link to parent. */
+	replace = ph_merge_children(phn, offset, cmp);
+	/* Set next/prev for sibling linked list. */
+	if (replace != NULL) {
+		if (parent != NULL) {
+			phn_prev_set(replace, parent, offset);
+			phn_lchild_set(parent, replace, offset);
+		} else {
+			phn_prev_set(replace, phn_prev_get(phn, offset),
+			    offset);
+			if (phn_prev_get(phn, offset) != NULL) {
+				phn_next_set(phn_prev_get(phn, offset), replace,
+				    offset);
+			}
+		}
+		phn_next_set(replace, phn_next_get(phn, offset), offset);
+		if (phn_next_get(phn, offset) != NULL) {
+			phn_prev_set(phn_next_get(phn, offset), replace,
+			    offset);
+		}
+	} else {
+		if (parent != NULL) {
+			void *next = phn_next_get(phn, offset);
+			phn_lchild_set(parent, next, offset);
+			if (next != NULL) {
+				phn_prev_set(next, parent, offset);
+			}
+		} else {
+			assert(phn_prev_get(phn, offset) != NULL);
+			phn_next_set(
+			    phn_prev_get(phn, offset),
+			    phn_next_get(phn, offset), offset);
+		}
+		if (phn_next_get(phn, offset) != NULL) {
+			phn_prev_set(
+			    phn_next_get(phn, offset),
+			    phn_prev_get(phn, offset), offset);
+		}
+	}
+}
+
+#define ph_structs(a_prefix, a_type)					\
+typedef struct {							\
+	phn_link_t link;						\
+} a_prefix##_link_t;							\
 									\
-	assert(a_phn0 != NULL);						\
+typedef struct {							\
-	assert(a_phn1 != NULL);						\
+	ph_t ph;							\
-	assert(a_cmp(a_phn0, a_phn1) <= 0);				\
+} a_prefix##_t;
-									\
-	phn_prev_set(a_type, a_field, a_phn1, a_phn0);			\
-	phn0child = phn_lchild_get(a_type, a_field, a_phn0);		\
-	phn_next_set(a_type, a_field, a_phn1, phn0child);		\
-	if (phn0child != NULL) {					\
-		phn_prev_set(a_type, a_field, phn0child, a_phn1);	\
-	}								\
-	phn_lchild_set(a_type, a_field, a_phn0, a_phn1);		\
-} while (0)
-
-#define phn_merge(a_type, a_field, a_phn0, a_phn1, a_cmp, r_phn) do {	\
-	if (a_phn0 == NULL) {						\
-		r_phn = a_phn1;						\
-	} else if (a_phn1 == NULL) {					\
-		r_phn = a_phn0;						\
-	} else if (a_cmp(a_phn0, a_phn1) < 0) {				\
-		phn_merge_ordered(a_type, a_field, a_phn0, a_phn1,	\
-		    a_cmp);						\
-		r_phn = a_phn0;						\
-	} else {							\
-		phn_merge_ordered(a_type, a_field, a_phn1, a_phn0,	\
-		    a_cmp);						\
-		r_phn = a_phn1;						\
-	}								\
-} while (0)
-
-#define ph_merge_siblings(a_type, a_field, a_phn, a_cmp, r_phn) do {	\
-	a_type *head = NULL;						\
-	a_type *tail = NULL;						\
-	a_type *phn0 = a_phn;						\
-	a_type *phn1 = phn_next_get(a_type, a_field, phn0);		\
-									\
-	/*								\
-	 * Multipass merge, wherein the first two elements of a FIFO	\
-	 * are repeatedly merged, and each result is appended to the	\
-	 * singly linked FIFO, until the FIFO contains only a single	\
-	 * element.  We start with a sibling list but no reference to	\
-	 * its tail, so we do a single pass over the sibling list to	\
-	 * populate the FIFO.						\
-	 */								\
-	if (phn1 != NULL) {						\
-		a_type *phnrest = phn_next_get(a_type, a_field, phn1);	\
-		if (phnrest != NULL) {					\
-			phn_prev_set(a_type, a_field, phnrest, NULL);	\
-		}							\
-		phn_prev_set(a_type, a_field, phn0, NULL);		\
-		phn_next_set(a_type, a_field, phn0, NULL);		\
-		phn_prev_set(a_type, a_field, phn1, NULL);		\
-		phn_next_set(a_type, a_field, phn1, NULL);		\
-		phn_merge(a_type, a_field, phn0, phn1, a_cmp, phn0);	\
-		head = tail = phn0;					\
-		phn0 = phnrest;						\
-		while (phn0 != NULL) {					\
-			phn1 = phn_next_get(a_type, a_field, phn0);	\
-			if (phn1 != NULL) {				\
-				phnrest = phn_next_get(a_type, a_field,	\
-				    phn1);				\
-				if (phnrest != NULL) {			\
-					phn_prev_set(a_type, a_field,	\
-					    phnrest, NULL);		\
-				}					\
-				phn_prev_set(a_type, a_field, phn0,	\
-				    NULL);				\
-				phn_next_set(a_type, a_field, phn0,	\
-				    NULL);				\
-				phn_prev_set(a_type, a_field, phn1,	\
-				    NULL);				\
-				phn_next_set(a_type, a_field, phn1,	\
-				    NULL);				\
-				phn_merge(a_type, a_field, phn0, phn1,	\
-				    a_cmp, phn0);			\
-				phn_next_set(a_type, a_field, tail,	\
-				    phn0);				\
-				tail = phn0;				\
-				phn0 = phnrest;				\
-			} else {					\
-				phn_next_set(a_type, a_field, tail,	\
-				    phn0);				\
-				tail = phn0;				\
-				phn0 = NULL;				\
-			}						\
-		}							\
-		phn0 = head;						\
-		phn1 = phn_next_get(a_type, a_field, phn0);		\
-		if (phn1 != NULL) {					\
-			while (true) {					\
-				head = phn_next_get(a_type, a_field,	\
-				    phn1);				\
-				assert(phn_prev_get(a_type, a_field,	\
-				    phn0) == NULL);			\
-				phn_next_set(a_type, a_field, phn0,	\
-				    NULL);				\
-				assert(phn_prev_get(a_type, a_field,	\
-				    phn1) == NULL);			\
-				phn_next_set(a_type, a_field, phn1,	\
-				    NULL);				\
-				phn_merge(a_type, a_field, phn0, phn1,	\
-				    a_cmp, phn0);			\
-				if (head == NULL) {			\
-					break;				\
-				}					\
-				phn_next_set(a_type, a_field, tail,	\
-				    phn0);				\
-				tail = phn0;				\
-				phn0 = head;				\
-				phn1 = phn_next_get(a_type, a_field,	\
-				    phn0);				\
-			}						\
-		}							\
-	}								\
-	r_phn = phn0;							\
-} while (0)
-
-#define ph_merge_aux(a_type, a_field, a_ph, a_cmp) do {			\
-	a_type *phn = phn_next_get(a_type, a_field, a_ph->ph_root);	\
-	if (phn != NULL) {						\
-		phn_prev_set(a_type, a_field, a_ph->ph_root, NULL);	\
-		phn_next_set(a_type, a_field, a_ph->ph_root, NULL);	\
-		phn_prev_set(a_type, a_field, phn, NULL);		\
-		ph_merge_siblings(a_type, a_field, phn, a_cmp, phn);	\
-		assert(phn_next_get(a_type, a_field, phn) == NULL);	\
-		phn_merge(a_type, a_field, a_ph->ph_root, phn, a_cmp,	\
-		    a_ph->ph_root);					\
-	}								\
-} while (0)
-
-#define ph_merge_children(a_type, a_field, a_phn, a_cmp, r_phn) do {	\
-	a_type *lchild = phn_lchild_get(a_type, a_field, a_phn);	\
-	if (lchild == NULL) {						\
-		r_phn = NULL;						\
-	} else {							\
-		ph_merge_siblings(a_type, a_field, lchild, a_cmp,	\
-		    r_phn);						\
-	}								\
-} while (0)
 
 /*
  * The ph_proto() macro generates function prototypes that correspond to the
  * functions generated by an equivalently parameterized call to ph_gen().
  */
-#define ph_proto(a_attr, a_prefix, a_ph_type, a_type)			\
+#define ph_proto(a_attr, a_prefix, a_type)				\
-a_attr void	a_prefix##new(a_ph_type *ph);				\
+									\
-a_attr bool	a_prefix##empty(a_ph_type *ph);				\
+a_attr void a_prefix##_new(a_prefix##_t *ph);				\
-a_attr a_type	*a_prefix##first(a_ph_type *ph);			\
+a_attr bool a_prefix##_empty(a_prefix##_t *ph);				\
-a_attr a_type	*a_prefix##any(a_ph_type *ph);				\
+a_attr a_type *a_prefix##_first(a_prefix##_t *ph);			\
-a_attr void	a_prefix##insert(a_ph_type *ph, a_type *phn);		\
+a_attr a_type *a_prefix##_any(a_prefix##_t *ph);			\
-a_attr a_type	*a_prefix##remove_first(a_ph_type *ph);			\
+a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn);		\
-a_attr a_type	*a_prefix##remove_any(a_ph_type *ph);			\
+a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph);		\
-a_attr void	a_prefix##remove(a_ph_type *ph, a_type *phn);
+a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph);			\
+a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn);
 
-/*
+/* The ph_gen() macro generates a type-specific pairing heap implementation. */
- * The ph_gen() macro generates a type-specific pairing heap implementation,
+#define ph_gen(a_attr, a_prefix, a_type, a_field, a_cmp)		\
- * based on the above cpp macros.
+JEMALLOC_ALWAYS_INLINE int						\
- */
+a_prefix##_ph_cmp(void *a, void *b) {					\
-#define ph_gen(a_attr, a_prefix, a_ph_type, a_type, a_field, a_cmp)	\
+	return a_cmp((a_type *)a, (a_type *)b);				\
-a_attr void								\
-a_prefix##new(a_ph_type *ph) {						\
-	memset(ph, 0, sizeof(ph(a_type)));				\
 }									\
+									\
+a_attr void								\
+a_prefix##_new(a_prefix##_t *ph) {					\
+	ph_new(&ph->ph);						\
+}									\
+									\
 a_attr bool								\
-a_prefix##empty(a_ph_type *ph) {					\
+a_prefix##_empty(a_prefix##_t *ph) {					\
-	return (ph->ph_root == NULL);					\
+	return ph_empty(&ph->ph);					\
 }									\
+									\
 a_attr a_type *								\
-a_prefix##first(a_ph_type *ph) {					\
+a_prefix##_first(a_prefix##_t *ph) {					\
-	if (ph->ph_root == NULL) {					\
+	return ph_first(&ph->ph, offsetof(a_type, a_field),		\
-		return NULL;						\
+	    &a_prefix##_ph_cmp);					\
-	}								\
-	ph_merge_aux(a_type, a_field, ph, a_cmp);			\
-	return ph->ph_root;						\
 }									\
+									\
 a_attr a_type *								\
-a_prefix##any(a_ph_type *ph) {						\
+a_prefix##_any(a_prefix##_t *ph) {					\
-	if (ph->ph_root == NULL) {					\
+	return ph_any(&ph->ph, offsetof(a_type, a_field));		\
-		return NULL;						\
-	}								\
-	a_type *aux = phn_next_get(a_type, a_field, ph->ph_root);	\
-	if (aux != NULL) {						\
-		return aux;						\
-	}								\
-	return ph->ph_root;						\
 }									\
+									\
 a_attr void								\
-a_prefix##insert(a_ph_type *ph, a_type *phn) {				\
+a_prefix##_insert(a_prefix##_t *ph, a_type *phn) {			\
-	memset(&phn->a_field, 0, sizeof(phn(a_type)));			\
+	ph_insert(&ph->ph, phn, offsetof(a_type, a_field));		\
-									\
-	/*								\
-	 * Treat the root as an aux list during insertion, and lazily	\
-	 * merge during a_prefix##remove_first().  For elements that	\
-	 * are inserted, then removed via a_prefix##remove() before the	\
-	 * aux list is ever processed, this makes insert/remove		\
-	 * constant-time, whereas eager merging would make insert	\
-	 * O(log n).							\
-	 */								\
-	if (ph->ph_root == NULL) {					\
-		ph->ph_root = phn;					\
-	} else {							\
-		phn_next_set(a_type, a_field, phn, phn_next_get(a_type,	\
-		    a_field, ph->ph_root));				\
-		if (phn_next_get(a_type, a_field, ph->ph_root) !=	\
-		    NULL) {						\
-			phn_prev_set(a_type, a_field,			\
-			    phn_next_get(a_type, a_field, ph->ph_root),	\
-			    phn);					\
-		}							\
-		phn_prev_set(a_type, a_field, phn, ph->ph_root);	\
-		phn_next_set(a_type, a_field, ph->ph_root, phn);	\
-	}								\
 }									\
+									\
 a_attr a_type *								\
-a_prefix##remove_first(a_ph_type *ph) {					\
+a_prefix##_remove_first(a_prefix##_t *ph) {				\
-	a_type *ret;							\
+	return ph_remove_first(&ph->ph, offsetof(a_type, a_field),	\
-									\
+	    a_prefix##_ph_cmp);						\
-	if (ph->ph_root == NULL) {					\
-		return NULL;						\
-	}								\
-	ph_merge_aux(a_type, a_field, ph, a_cmp);			\
-									\
-	ret = ph->ph_root;						\
-									\
-	ph_merge_children(a_type, a_field, ph->ph_root, a_cmp,		\
-	    ph->ph_root);						\
-									\
-	return ret;							\
 }									\
+									\
 a_attr a_type *								\
-a_prefix##remove_any(a_ph_type *ph) {					\
+a_prefix##_remove_any(a_prefix##_t *ph) {				\
-	/*								\
+	return ph_remove_any(&ph->ph, offsetof(a_type, a_field),	\
-	 * Remove the most recently inserted aux list element, or the	\
+	    a_prefix##_ph_cmp);						\
-	 * root if the aux list is empty.  This has the effect of	\
-	 * behaving as a LIFO (and insertion/removal is therefore	\
-	 * constant-time) if a_prefix##[remove_]first() are never	\
-	 * called.							\
-	 */								\
-	if (ph->ph_root == NULL) {					\
-		return NULL;						\
-	}								\
-	a_type *ret = phn_next_get(a_type, a_field, ph->ph_root);	\
-	if (ret != NULL) {						\
-		a_type *aux = phn_next_get(a_type, a_field, ret);	\
-		phn_next_set(a_type, a_field, ph->ph_root, aux);	\
-		if (aux != NULL) {					\
-			phn_prev_set(a_type, a_field, aux,		\
-			    ph->ph_root);				\
-		}							\
-		return ret;						\
-	}								\
-	ret = ph->ph_root;						\
-	ph_merge_children(a_type, a_field, ph->ph_root, a_cmp,		\
-	    ph->ph_root);						\
-	return ret;							\
 }									\
+									\
 a_attr void								\
-a_prefix##remove(a_ph_type *ph, a_type *phn) {				\
+a_prefix##_remove(a_prefix##_t *ph, a_type *phn) {			\
-	a_type *replace, *parent;					\
+	ph_remove(&ph->ph, phn, offsetof(a_type, a_field),		\
-									\
+	    a_prefix##_ph_cmp);						\
-	if (ph->ph_root == phn) {					\
-		/*							\
-		 * We can delete from aux list without merging it, but	\
-		 * we need to merge if we are dealing with the root	\
-		 * node and it has children.				\
-		 */							\
-		if (phn_lchild_get(a_type, a_field, phn) == NULL) {	\
-			ph->ph_root = phn_next_get(a_type, a_field,	\
-			    phn);					\
-			if (ph->ph_root != NULL) {			\
-				phn_prev_set(a_type, a_field,		\
-				    ph->ph_root, NULL);			\
-			}						\
-			return;						\
-		}							\
-		ph_merge_aux(a_type, a_field, ph, a_cmp);		\
-		if (ph->ph_root == phn) {				\
-			ph_merge_children(a_type, a_field, ph->ph_root,	\
-			    a_cmp, ph->ph_root);			\
-			return;						\
-		}							\
-	}								\
-									\
-	/* Get parent (if phn is leftmost child) before mutating. */	\
-	if ((parent = phn_prev_get(a_type, a_field, phn)) != NULL) {	\
-		if (phn_lchild_get(a_type, a_field, parent) != phn) {	\
-			parent = NULL;					\
-		}							\
-	}								\
-	/* Find a possible replacement node, and link to parent. */	\
-	ph_merge_children(a_type, a_field, phn, a_cmp, replace);	\
-	/* Set next/prev for sibling linked list. */			\
-	if (replace != NULL) {						\
-		if (parent != NULL) {					\
-			phn_prev_set(a_type, a_field, replace, parent);	\
-			phn_lchild_set(a_type, a_field, parent,		\
-			    replace);					\
-		} else {						\
-			phn_prev_set(a_type, a_field, replace,		\
-			    phn_prev_get(a_type, a_field, phn));	\
-			if (phn_prev_get(a_type, a_field, phn) !=	\
-			    NULL) {					\
-				phn_next_set(a_type, a_field,		\
-				    phn_prev_get(a_type, a_field, phn),	\
-				    replace);				\
-			}						\
-		}							\
-		phn_next_set(a_type, a_field, replace,			\
-		    phn_next_get(a_type, a_field, phn));		\
-		if (phn_next_get(a_type, a_field, phn) != NULL) {	\
-			phn_prev_set(a_type, a_field,			\
-			    phn_next_get(a_type, a_field, phn),		\
-			    replace);					\
-		}							\
-	} else {							\
-		if (parent != NULL) {					\
-			a_type *next = phn_next_get(a_type, a_field,	\
-			    phn);					\
-			phn_lchild_set(a_type, a_field, parent, next);	\
-			if (next != NULL) {				\
-				phn_prev_set(a_type, a_field, next,	\
-				    parent);				\
-			}						\
-		} else {						\
-			assert(phn_prev_get(a_type, a_field, phn) !=	\
-			    NULL);					\
-			phn_next_set(a_type, a_field,			\
-			    phn_prev_get(a_type, a_field, phn),		\
-			    phn_next_get(a_type, a_field, phn));	\
-		}							\
-		if (phn_next_get(a_type, a_field, phn) != NULL) {	\
-			phn_prev_set(a_type, a_field,			\
-			    phn_next_get(a_type, a_field, phn),		\
-			    phn_prev_get(a_type, a_field, phn));	\
-		}							\
-	}								\
 }
 
 #endif /* JEMALLOC_INTERNAL_PH_H */

src/edata.c

@@ -1,6 +1,6 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/jemalloc_internal_includes.h"
 
-ph_gen(, edata_avail_, edata_avail_t, edata_t, ph_link,
+ph_gen(, edata_avail, edata_t, avail_link,
     edata_esnead_comp)
-ph_gen(, edata_heap_, edata_heap_t, edata_t, ph_link, edata_snad_comp)
+ph_gen(, edata_heap, edata_t, heap_link, edata_snad_comp)

src/hpdata.c

@@ -15,7 +15,7 @@ hpdata_age_comp(const hpdata_t *a, const hpdata_t *b) {
 	return (a_age > b_age) - (a_age < b_age);
 }
 
-ph_gen(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t, ph_link, hpdata_age_comp)
+ph_gen(, hpdata_age_heap, hpdata_t, age_link, hpdata_age_comp)
 
 void
 hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {

test/unit/ph.c

@@ -3,11 +3,12 @@
 #include "jemalloc/internal/ph.h"
 
 typedef struct node_s node_t;
+ph_structs(heap, node_t);
 
 struct node_s {
 #define NODE_MAGIC 0x9823af7e
 	uint32_t magic;
-	phn(node_t) link;
+	heap_link_t link;
 	uint64_t key;
 };
 
@@ -36,8 +37,22 @@ node_cmp_magic(const node_t *a, const node_t *b) {
 	return node_cmp(a, b);
 }
 
-typedef ph(node_t) heap_t;
+ph_gen(static, heap, node_t, link, node_cmp_magic);
-ph_gen(static, heap_, heap_t, node_t, link, node_cmp_magic);
+
+static node_t *
+node_next_get(const node_t *node) {
+	return phn_next_get((node_t *)node, offsetof(node_t, link));
+}
+
+static node_t *
+node_prev_get(const node_t *node) {
+	return phn_prev_get((node_t *)node, offsetof(node_t, link));
+}
+
+static node_t *
+node_lchild_get(const node_t *node) {
+	return phn_lchild_get((node_t *)node, offsetof(node_t, link));
+}
 
 static void
 node_print(const node_t *node, unsigned depth) {
@@ -49,14 +64,14 @@ node_print(const node_t *node, unsigned depth) {
 	}
 	malloc_printf("%2"FMTu64"\n", node->key);
 
-	leftmost_child = phn_lchild_get(node_t, link, node);
+	leftmost_child = node_lchild_get(node);
 	if (leftmost_child == NULL) {
 		return;
 	}
 	node_print(leftmost_child, depth + 1);
 
-	for (sibling = phn_next_get(node_t, link, leftmost_child); sibling !=
+	for (sibling = node_next_get(leftmost_child); sibling !=
-	    NULL; sibling = phn_next_get(node_t, link, sibling)) {
+	    NULL; sibling = node_next_get(sibling)) {
 		node_print(sibling, depth + 1);
 	}
 }
@@ -66,16 +81,15 @@ heap_print(const heap_t *heap) {
 	node_t *auxelm;
 
 	malloc_printf("vvv heap %p vvv\n", heap);
-	if (heap->ph_root == NULL) {
+	if (heap->ph.root == NULL) {
 		goto label_return;
 	}
 
-	node_print(heap->ph_root, 0);
+	node_print(heap->ph.root, 0);
 
-	for (auxelm = phn_next_get(node_t, link, heap->ph_root); auxelm != NULL;
+	for (auxelm = node_next_get(heap->ph.root); auxelm != NULL;
-	    auxelm = phn_next_get(node_t, link, auxelm)) {
+	    auxelm = node_next_get(auxelm)) {
-		expect_ptr_eq(phn_next_get(node_t, link, phn_prev_get(node_t,
+		expect_ptr_eq(node_next_get(node_prev_get(auxelm)), auxelm,
-		    link, auxelm)), auxelm,
 		    "auxelm's prev doesn't link to auxelm");
 		node_print(auxelm, 0);
 	}
@@ -94,18 +108,17 @@ node_validate(const node_t *node, const node_t *parent) {
 		    "Child is less than parent");
 	}
 
-	leftmost_child = phn_lchild_get(node_t, link, node);
+	leftmost_child = node_lchild_get(node);
 	if (leftmost_child == NULL) {
 		return nnodes;
 	}
-	expect_ptr_eq((void *)phn_prev_get(node_t, link, leftmost_child),
+	expect_ptr_eq(node_prev_get(leftmost_child),
 	    (void *)node, "Leftmost child does not link to node");
 	nnodes += node_validate(leftmost_child, node);
 
-	for (sibling = phn_next_get(node_t, link, leftmost_child); sibling !=
+	for (sibling = node_next_get(leftmost_child); sibling !=
-	    NULL; sibling = phn_next_get(node_t, link, sibling)) {
+	    NULL; sibling = node_next_get(sibling)) {
-		expect_ptr_eq(phn_next_get(node_t, link, phn_prev_get(node_t,
+		expect_ptr_eq(node_next_get(node_prev_get(sibling)), sibling,
-		    link, sibling)), sibling,
 		    "sibling's prev doesn't link to sibling");
 		nnodes += node_validate(sibling, node);
 	}
@@ -117,16 +130,15 @@ heap_validate(const heap_t *heap) {
 	unsigned nnodes = 0;
 	node_t *auxelm;
 
-	if (heap->ph_root == NULL) {
+	if (heap->ph.root == NULL) {
 		goto label_return;
 	}
 
-	nnodes += node_validate(heap->ph_root, NULL);
+	nnodes += node_validate(heap->ph.root, NULL);
 
-	for (auxelm = phn_next_get(node_t, link, heap->ph_root); auxelm != NULL;
+	for (auxelm = node_next_get(heap->ph.root); auxelm != NULL;
-	    auxelm = phn_next_get(node_t, link, auxelm)) {
+	    auxelm = node_next_get(auxelm)) {
-		expect_ptr_eq(phn_next_get(node_t, link, phn_prev_get(node_t,
+		expect_ptr_eq(node_next_get(node_prev_get(auxelm)), auxelm,
-		    link, auxelm)), auxelm,
 		    "auxelm's prev doesn't link to auxelm");
 		nnodes += node_validate(auxelm, NULL);
 	}