project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor/fix ph.

Browse Source 
Refactor ph to support configurable comparison functions.  Use a cpp
macro code generation form equivalent to the rb macros so that pairing
heaps can be used for both run heaps and chunk heaps.

Remove per node parent pointers, and instead use leftmost siblings' prev
pointers to track parents.

Fix multi-pass sibling merging to iterate over intermediate results
using a FIFO, rather than a LIFO.  Use this fixed sibling merging
implementation for both merge phases of the auxiliary twopass algorithm
(first merging the aux list, then replacing the root with its merged
children).  This fixes both degenerate merge behavior and the potential
for deep recursion.

This regression was introduced by
6bafa6678f (Pairing heap).

This resolves #371.
This commit is contained in:
Jason Evans
2016-03-26 17:30:37 -07:00
parent 2ee2f1ec57
commit c6a2c39404
8 changed files with 613 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

552
include/jemalloc/internal/ph.h
View File

@@ -4,257 +4,341 @@
* "The Pairing Heap: A New Form of Self-Adjusting Heap"
* https://www.cs.cmu.edu/~sleator/papers/pairing-heaps.pdf
*
* With auxiliary list, described in a follow on paper
* With auxiliary twopass list, described in a follow on paper.
*
* "Pairing Heaps: Experiments and Analysis"
* http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.106.2988&rep=rep1&type=pdf
*
* Where search/nsearch/last are not needed, ph.h outperforms rb.h by ~7x fewer
* cpu cycles, and ~4x fewer memory references.
*
* Tagging parent/prev pointers on the next list was also described in the
* original paper, such that only two pointers are needed. This is not
* implemented here, as it substantially increases the memory references
* needed when ph_remove is called, almost overshadowing the other performance
* gains.
*
*******************************************************************************
*/
#ifdef JEMALLOC_H_TYPES
typedef struct ph_node_s ph_node_t;
typedef struct ph_heap_s ph_heap_t;
#ifndef PH_H_
#define PH_H_
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
struct ph_node_s {
ph_node_t *subheaps;
ph_node_t *parent;
ph_node_t *next;
ph_node_t *prev;
};
struct ph_heap_s {
ph_node_t *root;
};
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
ph_node_t *ph_merge_ordered(ph_node_t *heap1, ph_node_t *heap2);
ph_node_t *ph_merge(ph_node_t *heap1, ph_node_t *heap2);
ph_node_t *ph_merge_pairs(ph_node_t *subheaps);
void ph_merge_aux_list(ph_heap_t *l);
void ph_new(ph_heap_t *n);
ph_node_t *ph_first(ph_heap_t *l);
void ph_insert(ph_heap_t *l, ph_node_t *n);
ph_node_t *ph_remove_first(ph_heap_t *l);
void ph_remove(ph_heap_t *l, ph_node_t *n);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PH_C_))
/* Helper routines ************************************************************/
JEMALLOC_INLINE ph_node_t *
ph_merge_ordered(ph_node_t *heap1, ph_node_t *heap2)
{
assert(heap1 != NULL);
assert(heap2 != NULL);
assert ((uintptr_t)heap1 <= (uintptr_t)heap2);
heap2->parent = heap1;
heap2->prev = NULL;
heap2->next = heap1->subheaps;
if (heap1->subheaps != NULL)
heap1->subheaps->prev = heap2;
heap1->subheaps = heap2;
return (heap1);
/* Node structure. */
#define phn(a_type) \
struct { \
a_type *phn_prev; \
a_type *phn_next; \
a_type *phn_lchild; \
}
JEMALLOC_INLINE ph_node_t *
ph_merge(ph_node_t *heap1, ph_node_t *heap2)
{
if (heap1 == NULL)
return (heap2);
if (heap2 == NULL)
return (heap1);
/* Optional: user-settable comparison function */
if ((uintptr_t)heap1 < (uintptr_t)heap2)
return (ph_merge_ordered(heap1, heap2));
else
return (ph_merge_ordered(heap2, heap1));
/* Root structure. */
#define ph(a_type) \
struct { \
a_type *ph_root; \
}
JEMALLOC_INLINE ph_node_t *
ph_merge_pairs(ph_node_t *subheaps)
{
/* Internal utility macros. */
#define phn_lchild_get(a_type, a_field, a_phn) \
(a_phn->a_field.phn_lchild)
#define phn_lchild_set(a_type, a_field, a_phn, a_lchild) do { \
a_phn->a_field.phn_lchild = a_lchild; \
} while (0)
if (subheaps == NULL)
return (NULL);
if (subheaps->next == NULL)
return (subheaps);
{
ph_node_t *l0 = subheaps;
ph_node_t *l1 = l0->next;
ph_node_t *lrest = l1->next;
#define phn_next_get(a_type, a_field, a_phn) \
(a_phn->a_field.phn_next)
#define phn_prev_set(a_type, a_field, a_phn, a_prev) do { \
a_phn->a_field.phn_prev = a_prev; \
} while (0)
if (lrest != NULL)
lrest->prev = NULL;
l1->next = NULL;
l1->prev = NULL;
l0->next = NULL;
l0->prev = NULL;
return (ph_merge(ph_merge(l0, l1), ph_merge_pairs(lrest)));
}
}
#define phn_prev_get(a_type, a_field, a_phn) \
(a_phn->a_field.phn_prev)
#define phn_next_set(a_type, a_field, a_phn, a_next) do { \
a_phn->a_field.phn_next = a_next; \
} while (0)
#define phn_merge_ordered(a_type, a_field, a_phn0, a_phn1, a_cmp) do { \
a_type *phn0child; \
\
assert(a_phn0 != NULL); \
assert(a_phn1 != NULL); \
assert(a_cmp(a_phn0, a_phn1) <= 0); \
\
phn_prev_set(a_type, a_field, a_phn1, a_phn0); \
phn0child = phn_lchild_get(a_type, a_field, a_phn0); \
phn_next_set(a_type, a_field, a_phn1, phn0child); \
if (phn0child != NULL) \
phn_prev_set(a_type, a_field, phn0child, a_phn1); \
phn_lchild_set(a_type, a_field, a_phn0, a_phn1); \
} while (0)
#define phn_merge(a_type, a_field, a_phn0, a_phn1, a_cmp, r_phn) do { \
if (a_phn0 == NULL) \
r_phn = a_phn1; \
else if (a_phn1 == NULL) \
r_phn = a_phn0; \
else if (a_cmp(a_phn0, a_phn1) < 0) { \
phn_merge_ordered(a_type, a_field, a_phn0, a_phn1, \
a_cmp); \
r_phn = a_phn0; \
} else { \
phn_merge_ordered(a_type, a_field, a_phn1, a_phn0, \
a_cmp); \
r_phn = a_phn1; \
} \
} while (0)
#define ph_merge_siblings(a_type, a_field, a_phn, a_cmp, r_phn) do { \
a_type *head = NULL; \
a_type *tail = NULL; \
a_type *phn0 = a_phn; \
a_type *phn1 = phn_next_get(a_type, a_field, phn0); \
\
/* \
* Multipass merge, wherein the first two elements of a FIFO \
* are repeatedly merged, and each result is appended to the \
* singly linked FIFO, until the FIFO contains only a single \
* element. We start with a sibling list but no reference to \
* its tail, so we do a single pass over the sibling list to \
* populate the FIFO. \
*/ \
if (phn1 != NULL) { \
a_type *phnrest = phn_next_get(a_type, a_field, phn1); \
if (phnrest != NULL) \
phn_prev_set(a_type, a_field, phnrest, NULL); \
phn_prev_set(a_type, a_field, phn0, NULL); \
phn_next_set(a_type, a_field, phn0, NULL); \
phn_prev_set(a_type, a_field, phn1, NULL); \
phn_next_set(a_type, a_field, phn1, NULL); \
phn_merge(a_type, a_field, phn0, phn1, a_cmp, phn0); \
head = tail = phn0; \
phn0 = phnrest; \
while (phn0 != NULL) { \
phn1 = phn_next_get(a_type, a_field, phn0); \
if (phn1 != NULL) { \
phnrest = phn_next_get(a_type, a_field, \
phn1); \
if (phnrest != NULL) { \
phn_prev_set(a_type, a_field, \
phnrest, NULL); \
} \
phn_prev_set(a_type, a_field, phn0, \
NULL); \
phn_next_set(a_type, a_field, phn0, \
NULL); \
phn_prev_set(a_type, a_field, phn1, \
NULL); \
phn_next_set(a_type, a_field, phn1, \
NULL); \
phn_merge(a_type, a_field, phn0, phn1, \
a_cmp, phn0); \
phn_next_set(a_type, a_field, tail, \
phn0); \
tail = phn0; \
phn0 = phnrest; \
} else { \
phn_next_set(a_type, a_field, tail, \
phn0); \
tail = phn0; \
phn0 = NULL; \
} \
} \
phn0 = head; \
phn1 = phn_next_get(a_type, a_field, phn0); \
if (phn1 != NULL) { \
while (true) { \
head = phn_next_get(a_type, a_field, \
phn1); \
assert(phn_prev_get(a_type, a_field, \
phn0) == NULL); \
phn_next_set(a_type, a_field, phn0, \
NULL); \
assert(phn_prev_get(a_type, a_field, \
phn1) == NULL); \
phn_next_set(a_type, a_field, phn1, \
NULL); \
phn_merge(a_type, a_field, phn0, phn1, \
a_cmp, phn0); \
if (head == NULL) \
break; \
phn_next_set(a_type, a_field, tail, \
phn0); \
tail = phn0; \
phn0 = head; \
phn1 = phn_next_get(a_type, a_field, \
phn0); \
} \
} \
} \
r_phn = phn0; \
} while (0)
#define ph_merge_aux(a_type, a_field, a_ph, a_cmp) do { \
a_type *phn = phn_next_get(a_type, a_field, a_ph->ph_root); \
if (phn != NULL) { \
phn_prev_set(a_type, a_field, a_ph->ph_root, NULL); \
phn_next_set(a_type, a_field, a_ph->ph_root, NULL); \
phn_prev_set(a_type, a_field, phn, NULL); \
ph_merge_siblings(a_type, a_field, phn, a_cmp, phn); \
assert(phn_next_get(a_type, a_field, phn) == NULL); \
phn_merge(a_type, a_field, a_ph->ph_root, phn, a_cmp, \
a_ph->ph_root); \
} \
} while (0)
#define ph_merge_children(a_type, a_field, a_phn, a_cmp, r_phn) do { \
a_type *lchild = phn_lchild_get(a_type, a_field, a_phn); \
if (lchild == NULL) \
r_phn = NULL; \
else { \
ph_merge_siblings(a_type, a_field, lchild, a_cmp, \
r_phn); \
} \
} while (0)
/*
* Merge the aux list into the root node.
* The ph_proto() macro generates function prototypes that correspond to the
* functions generated by an equivalently parameterized call to ph_gen().
*/
JEMALLOC_INLINE void
ph_merge_aux_list(ph_heap_t *l)
{
#define ph_proto(a_attr, a_prefix, a_ph_type, a_type) \
a_attr void a_prefix##new(a_ph_type *ph); \
a_attr bool a_prefix##empty(a_ph_type *ph); \
a_attr a_type *a_prefix##first(a_ph_type *ph); \
a_attr void a_prefix##insert(a_ph_type *ph, a_type *phn); \
a_attr a_type *a_prefix##remove_first(a_ph_type *ph); \
a_attr void a_prefix##remove(a_ph_type *ph, a_type *phn);
if (l->root == NULL)
return;
if (l->root->next != NULL) {
ph_node_t *l0 = l->root->next;
ph_node_t *l1 = l0->next;
ph_node_t *lrest = NULL;
/* Multipass merge. */
while (l1 != NULL) {
lrest = l1->next;
if (lrest != NULL)
lrest->prev = NULL;
l1->next = NULL;
l1->prev = NULL;
l0->next = NULL;
l0->prev = NULL;
l0 = ph_merge(l0, l1);
l1 = lrest;
}
l->root->next = NULL;
l->root = ph_merge(l->root, l0);
}
/*
* The ph_gen() macro generates a type-specific pairing heap implementation,
* based on the above cpp macros.
*/
#define ph_gen(a_attr, a_prefix, a_ph_type, a_type, a_field, a_cmp) \
a_attr void \
a_prefix##new(a_ph_type *ph) \
{ \
\
memset(ph, 0, sizeof(ph(a_type))); \
} \
a_attr bool \
a_prefix##empty(a_ph_type *ph) { \
\
return (ph->ph_root == NULL); \
} \
a_attr a_type * \
a_prefix##first(a_ph_type *ph) \
{ \
\
if (ph->ph_root == NULL) \
return (NULL); \
ph_merge_aux(a_type, a_field, ph, a_cmp); \
return (ph->ph_root); \
} \
a_attr void \
a_prefix##insert(a_ph_type *ph, a_type *phn) \
{ \
\
memset(&phn->a_field, 0, sizeof(phn(a_type))); \
\
/* \
* Treat the root as an aux list during insertion, and lazily \
* merge during a_prefix##remove_first(). For elements that \
* are inserted, then removed via a_prefix##remove() before the \
* aux list is ever processed, this makes insert/remove \
* constant-time, whereas eager merging would make insert \
* O(log n). \
*/ \
if (ph->ph_root == NULL) \
ph->ph_root = phn; \
else { \
phn_next_set(a_type, a_field, phn, phn_next_get(a_type, \
a_field, ph->ph_root)); \
if (phn_next_get(a_type, a_field, ph->ph_root) != \
NULL) { \
phn_prev_set(a_type, a_field, \
phn_next_get(a_type, a_field, ph->ph_root), \
phn); \
} \
phn_prev_set(a_type, a_field, phn, ph->ph_root); \
phn_next_set(a_type, a_field, ph->ph_root, phn); \
} \
} \
a_attr a_type * \
a_prefix##remove_first(a_ph_type *ph) \
{ \
a_type *ret; \
\
if (ph->ph_root == NULL) \
return (NULL); \
ph_merge_aux(a_type, a_field, ph, a_cmp); \
\
ret = ph->ph_root; \
\
ph_merge_children(a_type, a_field, ph->ph_root, a_cmp, \
ph->ph_root); \
\
return (ret); \
} \
a_attr void \
a_prefix##remove(a_ph_type *ph, a_type *phn) \
{ \
a_type *replace, *parent; \
\
/* \
* We can delete from aux list without merging it, but we need \
* to merge if we are dealing with the root node. \
*/ \
if (ph->ph_root == phn) { \
ph_merge_aux(a_type, a_field, ph, a_cmp); \
if (ph->ph_root == phn) { \
ph_merge_children(a_type, a_field, ph->ph_root, \
a_cmp, ph->ph_root); \
return; \
} \
} \
\
/* Get parent (if phn is leftmost child) before mutating. */ \
if ((parent = phn_prev_get(a_type, a_field, phn)) != NULL) { \
if (phn_lchild_get(a_type, a_field, parent) != phn) \
parent = NULL; \
} \
/* Find a possible replacement node, and link to parent. */ \
ph_merge_children(a_type, a_field, phn, a_cmp, replace); \
/* Set next/prev for sibling linked list. */ \
if (replace != NULL) { \
if (parent != NULL) { \
phn_prev_set(a_type, a_field, replace, parent); \
phn_lchild_set(a_type, a_field, parent, \
replace); \
} else { \
phn_prev_set(a_type, a_field, replace, \
phn_prev_get(a_type, a_field, phn)); \
if (phn_prev_get(a_type, a_field, phn) != \
NULL) { \
phn_next_set(a_type, a_field, \
phn_prev_get(a_type, a_field, phn), \
replace); \
} \
} \
phn_next_set(a_type, a_field, replace, \
phn_next_get(a_type, a_field, phn)); \
if (phn_next_get(a_type, a_field, phn) != NULL) { \
phn_prev_set(a_type, a_field, \
phn_next_get(a_type, a_field, phn), \
replace); \
} \
} else { \
if (parent != NULL) { \
a_type *next = phn_next_get(a_type, a_field, \
phn); \
phn_lchild_set(a_type, a_field, parent, next); \
if (next != NULL) { \
phn_prev_set(a_type, a_field, next, \
parent); \
} \
} else { \
assert(phn_prev_get(a_type, a_field, phn) != \
NULL); \
phn_next_set(a_type, a_field, \
phn_prev_get(a_type, a_field, phn), \
phn_next_get(a_type, a_field, phn)); \
} \
if (phn_next_get(a_type, a_field, phn) != NULL) { \
phn_prev_set(a_type, a_field, \
phn_next_get(a_type, a_field, phn), \
phn_prev_get(a_type, a_field, phn)); \
} \
} \
}
/* User API *******************************************************************/
JEMALLOC_INLINE void
ph_new(ph_heap_t *n)
{
memset(n, 0, sizeof(ph_heap_t));
}
JEMALLOC_INLINE ph_node_t *
ph_first(ph_heap_t *l)
{
/*
* For the cost of an extra pointer, a l->min could be stored instead of
* merging the aux list here. Current users always call ph_remove(l,
* ph_first(l)) though, and the aux list must always be merged for
* delete of the min node anyway.
*/
ph_merge_aux_list(l);
return (l->root);
}
JEMALLOC_INLINE void
ph_insert(ph_heap_t *l, ph_node_t *n)
{
memset(n, 0, sizeof(ph_node_t));
/*
* Non-aux list insert:
*
* l->root = ph_merge(l->root, n);
*
* Aux list insert:
*/
if (l->root == NULL)
l->root = n;
else {
n->next = l->root->next;
if (l->root->next != NULL)
l->root->next->prev = n;
n->prev = l->root;
l->root->next = n;
}
}
JEMALLOC_INLINE ph_node_t *
ph_remove_first(ph_heap_t *l)
{
ph_node_t *ret;
ph_merge_aux_list(l);
if (l->root == NULL)
return (NULL);
ret = l->root;
l->root = ph_merge_pairs(l->root->subheaps);
return (ret);
}
JEMALLOC_INLINE void
ph_remove(ph_heap_t *l, ph_node_t *n)
{
ph_node_t *replace;
/*
* We can delete from aux list without merging it, but we need to merge
* if we are dealing with the root node.
*/
if (l->root == n) {
ph_merge_aux_list(l);
if (l->root == n) {
ph_remove_first(l);
return;
}
}
/* Find a possible replacement node, and link to parent. */
replace = ph_merge_pairs(n->subheaps);
if (n->parent != NULL && n->parent->subheaps == n) {
if (replace != NULL)
n->parent->subheaps = replace;
else
n->parent->subheaps = n->next;
}
/* Set next/prev for sibling linked list. */
if (replace != NULL) {
replace->parent = n->parent;
replace->prev = n->prev;
if (n->prev != NULL)
n->prev->next = replace;
replace->next = n->next;
if (n->next != NULL)
n->next->prev = replace;
} else {
if (n->prev != NULL)
n->prev->next = n->next;
if (n->next != NULL)
n->next->prev = n->prev;
}
}
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#endif /* PH_H_ */