Split rtree_elm_t into rtree_{node,leaf}_elm_t.

This allows leaf elements to differ in size from internal node elements. In principle it would be more correct to use a different type for each level of the tree, but due to implementation details related to atomic operations, we use casts anyway, thus counteracting the value of additional type correctness. Furthermore, such a scheme would require function code generation (via cpp macros), as well as either unwieldy type names for leaves or type aliases, e.g. typedef struct rtree_elm_d2_s rtree_leaf_elm_t; This alternate strategy would be more correct, and with less code duplication, but probably not worth the complexity.
2017-03-16 09:46:42 -07:00
parent f50d6009fe
commit 944c8a3383
9 changed files with 459 additions and 258 deletions
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -413,17 +413,19 @@ psz2ind
 psz2u
 rtree_clear
 rtree_delete
-rtree_elm_acquire
-rtree_elm_lookup
-rtree_elm_lookup_hard
-rtree_elm_read
-rtree_elm_read_acquired
-rtree_elm_release
-rtree_elm_witness_access
-rtree_elm_witness_acquire
-rtree_elm_witness_release
-rtree_elm_write
-rtree_elm_write_acquired
+rtree_leaf_alloc
+rtree_leaf_dalloc
+rtree_leaf_elm_acquire
+rtree_leaf_elm_lookup
+rtree_leaf_elm_lookup_hard
+rtree_leaf_elm_read
+rtree_leaf_elm_read_acquired
+rtree_leaf_elm_release
+rtree_leaf_elm_witness_access
+rtree_leaf_elm_witness_acquire
+rtree_leaf_elm_witness_release
+rtree_leaf_elm_write
+rtree_leaf_elm_write_acquired
 rtree_leafkey
 rtree_new
 rtree_node_alloc
@@ -513,7 +515,7 @@ tsd_prof_tdata_get
 tsd_prof_tdata_set
 tsd_prof_tdatap_get
 tsd_rtree_ctxp_get
-tsd_rtree_elm_witnessesp_get
+tsd_rtree_leaf_elm_witnessesp_get
 tsd_set
 tsd_tcache_enabled_get
 tsd_tcache_enabled_set
--- a/include/jemalloc/internal/rtree_externs.h
+++ b/include/jemalloc/internal/rtree_externs.h
@@ -8,36 +8,40 @@
 * level.
 */
 static const rtree_level_t rtree_levels[] = {
-#if RTREE_NSB <= 10
+#if RTREE_HEIGHT == 1
 	{RTREE_NSB, RTREE_NHIB + RTREE_NSB}
-#elif RTREE_NSB <= 36
+#elif RTREE_HEIGHT == 2
 	{RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2},
 	{RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB}
-#elif RTREE_NSB <= 52
+#elif RTREE_HEIGHT == 3
 	{RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3},
 	{RTREE_NSB/3 + RTREE_NSB%3/2,
 	    RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2},
 	{RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB}
 #else
-#  error Unsupported number of significant virtual address bits
+#  error Unsupported rtree height
 #endif
 };

 bool rtree_new(rtree_t *rtree);
 #ifdef JEMALLOC_JET
-typedef rtree_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t);
+typedef rtree_node_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t);
 extern rtree_node_alloc_t *rtree_node_alloc;
-typedef void (rtree_node_dalloc_t)(tsdn_t *, rtree_t *, rtree_elm_t *);
+typedef rtree_leaf_elm_t *(rtree_leaf_alloc_t)(tsdn_t *, rtree_t *, size_t);
+extern rtree_leaf_alloc_t *rtree_leaf_alloc;
+typedef void (rtree_node_dalloc_t)(tsdn_t *, rtree_t *, rtree_node_elm_t *);
 extern rtree_node_dalloc_t *rtree_node_dalloc;
+typedef void (rtree_leaf_dalloc_t)(tsdn_t *, rtree_t *, rtree_leaf_elm_t *);
+extern rtree_leaf_dalloc_t *rtree_leaf_dalloc;
 void rtree_delete(tsdn_t *tsdn, rtree_t *rtree);
 #endif
-rtree_elm_t *rtree_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,
+rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,
    rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
-void rtree_elm_witness_acquire(tsdn_t *tsdn, const rtree_t *rtree,
-    uintptr_t key, const rtree_elm_t *elm);
-void rtree_elm_witness_access(tsdn_t *tsdn, const rtree_t *rtree,
-    const rtree_elm_t *elm);
-void rtree_elm_witness_release(tsdn_t *tsdn, const rtree_t *rtree,
-    const rtree_elm_t *elm);
+void rtree_leaf_elm_witness_acquire(tsdn_t *tsdn, const rtree_t *rtree,
+    uintptr_t key, const rtree_leaf_elm_t *elm);
+void rtree_leaf_elm_witness_access(tsdn_t *tsdn, const rtree_t *rtree,
+    const rtree_leaf_elm_t *elm);
+void rtree_leaf_elm_witness_release(tsdn_t *tsdn, const rtree_t *rtree,
+    const rtree_leaf_elm_t *elm);

 #endif /* JEMALLOC_INTERNAL_RTREE_EXTERNS_H */
--- a/include/jemalloc/internal/rtree_inlines.h
+++ b/include/jemalloc/internal/rtree_inlines.h
@@ -4,21 +4,22 @@
 #ifndef JEMALLOC_ENABLE_INLINE
 uintptr_t rtree_leafkey(uintptr_t key);
 uintptr_t rtree_subkey(uintptr_t key, unsigned level);
-extent_t *rtree_elm_read(rtree_elm_t *elm, bool dependent);
-void rtree_elm_write(rtree_elm_t *elm, const extent_t *extent);
-rtree_elm_t *rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree,
+extent_t *rtree_leaf_elm_read(rtree_leaf_elm_t *elm, bool dependent);
+void rtree_leaf_elm_write(rtree_leaf_elm_t *elm, const extent_t *extent);
+rtree_leaf_elm_t *rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree,
    rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
 bool rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
    uintptr_t key, const extent_t *extent);
 extent_t *rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
    uintptr_t key, bool dependent);
-rtree_elm_t *rtree_elm_acquire(tsdn_t *tsdn, rtree_t *rtree,
+rtree_leaf_elm_t *rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree,
    rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
-extent_t *rtree_elm_read_acquired(tsdn_t *tsdn, const rtree_t *rtree,
-    rtree_elm_t *elm);
-void rtree_elm_write_acquired(tsdn_t *tsdn, const rtree_t *rtree,
-    rtree_elm_t *elm, const extent_t *extent);
-void rtree_elm_release(tsdn_t *tsdn, const rtree_t *rtree, rtree_elm_t *elm);
+extent_t *rtree_leaf_elm_read_acquired(tsdn_t *tsdn, const rtree_t *rtree,
+    rtree_leaf_elm_t *elm);
+void rtree_leaf_elm_write_acquired(tsdn_t *tsdn, const rtree_t *rtree,
+    rtree_leaf_elm_t *elm, const extent_t *extent);
+void rtree_leaf_elm_release(tsdn_t *tsdn, const rtree_t *rtree,
+    rtree_leaf_elm_t *elm);
 void rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
    uintptr_t key);
 #endif
@@ -45,7 +46,7 @@ rtree_subkey(uintptr_t key, unsigned level) {
 }

 JEMALLOC_ALWAYS_INLINE extent_t *
-rtree_elm_read(rtree_elm_t *elm, bool dependent) {
+rtree_leaf_elm_read(rtree_leaf_elm_t *elm, bool dependent) {
 	extent_t *extent;

 	if (dependent) {
@@ -55,7 +56,7 @@ rtree_elm_read(rtree_elm_t *elm, bool dependent) {
 		 * synchronization, because the rtree update became visible in
 		 * memory before the pointer came into existence.
 		 */
-		extent = (extent_t *)atomic_load_p(&elm->child_or_extent,
+		extent = (extent_t *)atomic_load_p(&elm->extent,
 		    ATOMIC_RELAXED);
 	} else {
 		/*
@@ -63,7 +64,7 @@ rtree_elm_read(rtree_elm_t *elm, bool dependent) {
 		 * dependent on a previous rtree write, which means a stale read
 		 * could result if synchronization were omitted here.
 		 */
-		extent = (extent_t *)atomic_load_p(&elm->child_or_extent,
+		extent = (extent_t *)atomic_load_p(&elm->extent,
 		    ATOMIC_ACQUIRE);
 	}

@@ -74,12 +75,12 @@ rtree_elm_read(rtree_elm_t *elm, bool dependent) {
 }

 JEMALLOC_INLINE void
-rtree_elm_write(rtree_elm_t *elm, const extent_t *extent) {
-	atomic_store_p(&elm->child_or_extent, (void *)extent, ATOMIC_RELEASE);
+rtree_leaf_elm_write(rtree_leaf_elm_t *elm, const extent_t *extent) {
+	atomic_store_p(&elm->extent, (void *)extent, ATOMIC_RELEASE);
 }

-JEMALLOC_ALWAYS_INLINE rtree_elm_t *
-rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
+JEMALLOC_ALWAYS_INLINE rtree_leaf_elm_t *
+rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
    uintptr_t key, bool dependent, bool init_missing) {
 	assert(key != 0);
 	assert(!dependent || !init_missing);
@@ -87,7 +88,7 @@ rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	uintptr_t leafkey = rtree_leafkey(key);
 #define RTREE_CACHE_CHECK(i) do {					\
 	if (likely(rtree_ctx->cache[i].leafkey == leafkey)) {		\
-		rtree_elm_t *leaf = rtree_ctx->cache[i].leaf;		\
+		rtree_leaf_elm_t *leaf = rtree_ctx->cache[i].leaf;	\
 		if (likely(leaf != NULL)) {				\
 			/* Reorder. */					\
 			memmove(&rtree_ctx->cache[1],			\
@@ -117,24 +118,24 @@ rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	}
 #undef RTREE_CACHE_CHECK

-	return rtree_elm_lookup_hard(tsdn, rtree, rtree_ctx, key, dependent,
-	    init_missing);
+	return rtree_leaf_elm_lookup_hard(tsdn, rtree, rtree_ctx, key,
+	    dependent, init_missing);
 }

 JEMALLOC_INLINE bool
 rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
    const extent_t *extent) {
-	rtree_elm_t *elm;
+	rtree_leaf_elm_t *elm;

 	assert(extent != NULL); /* Use rtree_clear() for this case. */
 	assert(((uintptr_t)extent & (uintptr_t)0x1) == (uintptr_t)0x0);

-	elm = rtree_elm_lookup(tsdn, rtree, rtree_ctx, key, false, true);
+	elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, key, false, true);
 	if (elm == NULL) {
 		return true;
 	}
-	assert(rtree_elm_read(elm, false) == NULL);
-	rtree_elm_write(elm, extent);
+	assert(rtree_leaf_elm_read(elm, false) == NULL);
+	rtree_leaf_elm_write(elm, extent);

 	return false;
 }
@@ -142,21 +143,22 @@ rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
 JEMALLOC_ALWAYS_INLINE extent_t *
 rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
    bool dependent) {
-	rtree_elm_t *elm;
+	rtree_leaf_elm_t *elm;

-	elm = rtree_elm_lookup(tsdn, rtree, rtree_ctx, key, dependent, false);
+	elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, key, dependent,
+	    false);
 	if (!dependent && elm == NULL) {
 		return NULL;
 	}

-	return rtree_elm_read(elm, dependent);
+	return rtree_leaf_elm_read(elm, dependent);
 }

-JEMALLOC_INLINE rtree_elm_t *
-rtree_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
+JEMALLOC_INLINE rtree_leaf_elm_t *
+rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
    uintptr_t key, bool dependent, bool init_missing) {
-	rtree_elm_t *elm = rtree_elm_lookup(tsdn, rtree, rtree_ctx, key,
-	    dependent, init_missing);
+	rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
+	    key, dependent, init_missing);
 	if (!dependent && elm == NULL) {
 		return NULL;
 	}
@@ -164,14 +166,14 @@ rtree_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	spin_t spinner = SPIN_INITIALIZER;
 	while (true) {
 		/* The least significant bit serves as a lock. */
-		void *extent_and_lock = atomic_load_p(&elm->child_or_extent,
+		void *extent_and_lock = atomic_load_p(&elm->extent,
 		    ATOMIC_RELAXED);
 		if (likely(((uintptr_t)extent_and_lock & (uintptr_t)0x1) == 0))
 		{
 			void *locked = (void *)((uintptr_t)extent_and_lock
 			    | (uintptr_t)0x1);
 			if (likely(atomic_compare_exchange_strong_p(
-			    &elm->child_or_extent, &extent_and_lock, locked,
+			    &elm->extent, &extent_and_lock, locked,
 			    ATOMIC_ACQUIRE, ATOMIC_RELAXED))) {
 				break;
 			}
@@ -180,58 +182,61 @@ rtree_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	}

 	if (config_debug) {
-		rtree_elm_witness_acquire(tsdn, rtree, key, elm);
+		rtree_leaf_elm_witness_acquire(tsdn, rtree, key, elm);
 	}

 	return elm;
 }

 JEMALLOC_INLINE extent_t *
-rtree_elm_read_acquired(tsdn_t *tsdn, const rtree_t *rtree, rtree_elm_t *elm) {
+rtree_leaf_elm_read_acquired(tsdn_t *tsdn, const rtree_t *rtree,
+    rtree_leaf_elm_t *elm) {
 	extent_t *extent;
-	void *ptr = atomic_load_p(&elm->child_or_extent, ATOMIC_RELAXED);
+	void *ptr = atomic_load_p(&elm->extent, ATOMIC_RELAXED);
 	assert(((uintptr_t)ptr & (uintptr_t)0x1) == (uintptr_t)0x1);
 	extent = (extent_t *)((uintptr_t)ptr & ~((uintptr_t)0x1));
 	assert(((uintptr_t)extent & (uintptr_t)0x1) == (uintptr_t)0x0);

 	if (config_debug) {
-		rtree_elm_witness_access(tsdn, rtree, elm);
+		rtree_leaf_elm_witness_access(tsdn, rtree, elm);
 	}

 	return extent;
 }

 JEMALLOC_INLINE void
-rtree_elm_write_acquired(tsdn_t *tsdn, const rtree_t *rtree, rtree_elm_t *elm,
-    const extent_t *extent) {
+rtree_leaf_elm_write_acquired(tsdn_t *tsdn, const rtree_t *rtree,
+    rtree_leaf_elm_t *elm, const extent_t *extent) {
 	assert(((uintptr_t)extent & (uintptr_t)0x1) == (uintptr_t)0x0);
-	assert(((uintptr_t)atomic_load_p(&elm->child_or_extent, ATOMIC_RELAXED)
+	assert(((uintptr_t)atomic_load_p(&elm->extent, ATOMIC_RELAXED)
 	    & (uintptr_t)0x1) == (uintptr_t)0x1);

 	if (config_debug) {
-		rtree_elm_witness_access(tsdn, rtree, elm);
+		rtree_leaf_elm_witness_access(tsdn, rtree, elm);
 	}
-	atomic_store_p(&elm->child_or_extent, (void *)((uintptr_t)extent
-	    | (uintptr_t)0x1), ATOMIC_RELEASE);
-	assert(rtree_elm_read_acquired(tsdn, rtree, elm) == extent);
+	atomic_store_p(&elm->extent, (void *)((uintptr_t)extent |
+	    (uintptr_t)0x1), ATOMIC_RELEASE);
+	assert(rtree_leaf_elm_read_acquired(tsdn, rtree, elm) == extent);
 }

 JEMALLOC_INLINE void
-rtree_elm_release(tsdn_t *tsdn, const rtree_t *rtree, rtree_elm_t *elm) {
-	rtree_elm_write(elm, rtree_elm_read_acquired(tsdn, rtree, elm));
+rtree_leaf_elm_release(tsdn_t *tsdn, const rtree_t *rtree,
+    rtree_leaf_elm_t *elm) {
+	rtree_leaf_elm_write(elm, rtree_leaf_elm_read_acquired(tsdn, rtree,
+	    elm));
 	if (config_debug) {
-		rtree_elm_witness_release(tsdn, rtree, elm);
+		rtree_leaf_elm_witness_release(tsdn, rtree, elm);
 	}
 }

 JEMALLOC_INLINE void
 rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
    uintptr_t key) {
-	rtree_elm_t *elm;
+	rtree_leaf_elm_t *elm;

-	elm = rtree_elm_acquire(tsdn, rtree, rtree_ctx, key, true, false);
-	rtree_elm_write_acquired(tsdn, rtree, elm, NULL);
-	rtree_elm_release(tsdn, rtree, elm);
+	elm = rtree_leaf_elm_acquire(tsdn, rtree, rtree_ctx, key, true, false);
+	rtree_leaf_elm_write_acquired(tsdn, rtree, elm, NULL);
+	rtree_leaf_elm_release(tsdn, rtree, elm);
 }
 #endif

--- a/include/jemalloc/internal/rtree_structs.h
+++ b/include/jemalloc/internal/rtree_structs.h
@@ -1,18 +1,21 @@
 #ifndef JEMALLOC_INTERNAL_RTREE_STRUCTS_H
 #define JEMALLOC_INTERNAL_RTREE_STRUCTS_H

-struct rtree_elm_s {
-	/* Either "rtree_elm_t *child;" or "extent_t *extent;". */
-	atomic_p_t	child_or_extent;
+struct rtree_node_elm_s {
+	atomic_p_t	child;
 };

-struct rtree_elm_witness_s {
-	const rtree_elm_t	*elm;
+struct rtree_leaf_elm_s {
+	atomic_p_t	extent;
+};
+
+struct rtree_leaf_elm_witness_s {
+	const rtree_leaf_elm_t	*elm;
 	witness_t		witness;
 };

-struct rtree_elm_witness_tsd_s {
-	rtree_elm_witness_t	witnesses[RTREE_ELM_ACQUIRE_MAX];
+struct rtree_leaf_elm_witness_tsd_s {
+	rtree_leaf_elm_witness_t	witnesses[RTREE_ELM_ACQUIRE_MAX];
 };

 struct rtree_level_s {
@@ -26,8 +29,8 @@ struct rtree_level_s {
 };

 struct rtree_ctx_cache_elm_s {
-	uintptr_t	leafkey;
-	rtree_elm_t	*leaf;
+	uintptr_t		leafkey;
+	rtree_leaf_elm_t	*leaf;
 };

 struct rtree_ctx_s {
@@ -38,7 +41,7 @@ struct rtree_ctx_s {
 };

 struct rtree_s {
-	/* An rtree_elm_t *. */
+	/* An rtree_{internal,leaf}_elm_t *. */
 	atomic_p_t	root;
 	malloc_mutex_t	init_lock;
 };
--- a/include/jemalloc/internal/rtree_types.h
+++ b/include/jemalloc/internal/rtree_types.h
@@ -8,9 +8,10 @@
 *******************************************************************************
 */

-typedef struct rtree_elm_s rtree_elm_t;
-typedef struct rtree_elm_witness_s rtree_elm_witness_t;
-typedef struct rtree_elm_witness_tsd_s rtree_elm_witness_tsd_t;
+typedef struct rtree_node_elm_s rtree_node_elm_t;
+typedef struct rtree_leaf_elm_s rtree_leaf_elm_t;
+typedef struct rtree_leaf_elm_witness_s rtree_leaf_elm_witness_t;
+typedef struct rtree_leaf_elm_witness_tsd_s rtree_leaf_elm_witness_tsd_t;
 typedef struct rtree_level_s rtree_level_t;
 typedef struct rtree_ctx_cache_elm_s rtree_ctx_cache_elm_t;
 typedef struct rtree_ctx_s rtree_ctx_t;
@@ -23,7 +24,15 @@ typedef struct rtree_s rtree_t;
 /* Number of significant bits. */
 #define RTREE_NSB (LG_VADDR - RTREE_NLIB)
 /* Number of levels in radix tree. */
-#define RTREE_HEIGHT (sizeof(rtree_levels)/sizeof(rtree_level_t))
+#if RTREE_NSB <= 10
+#  define RTREE_HEIGHT 1
+#elif RTREE_NSB <= 36
+#  define RTREE_HEIGHT 2
+#elif RTREE_NSB <= 52
+#  define RTREE_HEIGHT 3
+#else
+#  error Unsupported number of significant virtual address bits
+#endif

 /*
 * Number of leafkey/leaf pairs to cache.  Each entry supports an entire leaf,
@@ -47,16 +56,16 @@ typedef struct rtree_s rtree_t;

 /*
 * Maximum number of concurrently acquired elements per thread.  This controls
- * how many witness_t structures are embedded in tsd.  Ideally rtree_elm_t would
- * have a witness_t directly embedded, but that would dramatically bloat the
- * tree.  This must contain enough entries to e.g. coalesce two extents.
+ * how many witness_t structures are embedded in tsd.  Ideally rtree_leaf_elm_t
+ * would have a witness_t directly embedded, but that would dramatically bloat
+ * the tree.  This must contain enough entries to e.g. coalesce two extents.
 */
 #define RTREE_ELM_ACQUIRE_MAX	4

-/* Initializers for rtree_elm_witness_tsd_t. */
+/* Initializers for rtree_leaf_elm_witness_tsd_t. */
 #define RTREE_ELM_WITNESS_INITIALIZER {					\
 	NULL,								\
-	WITNESS_INITIALIZER("rtree_elm", WITNESS_RANK_RTREE_ELM)	\
+	WITNESS_INITIALIZER("rtree_leaf_elm", WITNESS_RANK_RTREE_ELM)	\
 }

 #define RTREE_ELM_WITNESS_TSD_INITIALIZER {				\
--- a/include/jemalloc/internal/tsd_structs.h
+++ b/include/jemalloc/internal/tsd_structs.h
@@ -29,7 +29,7 @@ struct tsd_init_head_s {
 						yes,	no)		\
    O(rtree_ctx,		rtree_ctx_t,	no,	no)		\
    O(witnesses,		witness_list_t,	no,	yes)		\
-    O(rtree_elm_witnesses,	rtree_elm_witness_tsd_t,		\
+    O(rtree_leaf_elm_witnesses,	rtree_leaf_elm_witness_tsd_t,		\
 						no,	no)		\
    O(witness_fork,		bool,		yes,	no)		\