Header refactoring: unify and de-catchall rtree module.

include/jemalloc/internal/arena_inlines_b.h

@@ -3,6 +3,7 @@
 
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/rtree.h"
 #include "jemalloc/internal/size_classes.h"
 #include "jemalloc/internal/ticker.h"
 

include/jemalloc/internal/extent_externs.h

@@ -4,6 +4,7 @@
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/ph.h"
 #include "jemalloc/internal/rb.h"
+#include "jemalloc/internal/rtree.h"
 
 extern rtree_t			extents_rtree;
 extern const extent_hooks_t	extent_hooks_default;

include/jemalloc/internal/jemalloc_internal_includes.h

@@ -44,7 +44,6 @@
 #include "jemalloc/internal/extent_dss_types.h"
 #include "jemalloc/internal/base_types.h"
 #include "jemalloc/internal/arena_types.h"
-#include "jemalloc/internal/rtree_types.h"
 #include "jemalloc/internal/tcache_types.h"
 #include "jemalloc/internal/prof_types.h"
 
@@ -59,7 +58,6 @@
 #include "jemalloc/internal/base_structs.h"
 #include "jemalloc/internal/prof_structs.h"
 #include "jemalloc/internal/arena_structs_b.h"
-#include "jemalloc/internal/rtree_structs.h"
 #include "jemalloc/internal/tcache_structs.h"
 #include "jemalloc/internal/background_thread_structs.h"
 
@@ -73,7 +71,6 @@
 #include "jemalloc/internal/extent_mmap_externs.h"
 #include "jemalloc/internal/base_externs.h"
 #include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/rtree_externs.h"
 #include "jemalloc/internal/large_externs.h"
 #include "jemalloc/internal/tcache_externs.h"
 #include "jemalloc/internal/prof_externs.h"
@@ -85,7 +82,6 @@
 
 #include "jemalloc/internal/mutex_pool_inlines.h"
 #include "jemalloc/internal/jemalloc_internal_inlines_a.h"
-#include "jemalloc/internal/rtree_inlines.h"
 #include "jemalloc/internal/base_inlines.h"
 /*
  * Include portions of arena code interleaved with tcache code in order to

include/jemalloc/internal/jemalloc_internal_inlines_b.h

@@ -1,6 +1,8 @@
 #ifndef JEMALLOC_INTERNAL_INLINES_B_H
 #define JEMALLOC_INTERNAL_INLINES_B_H
 
+#include "jemalloc/internal/rtree.h"
+
 /* Choose an arena based on a per-thread value. */
 static inline arena_t *
 arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {

include/jemalloc/internal/rtree.h

@@ -1,8 +1,132 @@
-#ifndef JEMALLOC_INTERNAL_RTREE_INLINES_H
+#ifndef JEMALLOC_INTERNAL_RTREE_H
-#define JEMALLOC_INTERNAL_RTREE_INLINES_H
+#define JEMALLOC_INTERNAL_RTREE_H
 
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/rtree_tsd.h"
 #include "jemalloc/internal/size_classes.h"
-#include "jemalloc/internal/spin.h"
+#include "jemalloc/internal/tsd.h"
+
+/*
+ * This radix tree implementation is tailored to the singular purpose of
+ * associating metadata with extents that are currently owned by jemalloc.
+ *
+ *******************************************************************************
+ */
+
+/* Number of high insignificant bits. */
+#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR)
+/* Number of low insigificant bits. */
+#define RTREE_NLIB LG_PAGE
+/* Number of significant bits. */
+#define RTREE_NSB (LG_VADDR - RTREE_NLIB)
+/* Number of levels in radix tree. */
+#if RTREE_NSB <= 10
+#  define RTREE_HEIGHT 1
+#elif RTREE_NSB <= 36
+#  define RTREE_HEIGHT 2
+#elif RTREE_NSB <= 52
+#  define RTREE_HEIGHT 3
+#else
+#  error Unsupported number of significant virtual address bits
+#endif
+/* Use compact leaf representation if virtual address encoding allows. */
+#if RTREE_NHIB >= LG_CEIL_NSIZES
+#  define RTREE_LEAF_COMPACT
+#endif
+
+/* Needed for initialization only. */
+#define RTREE_LEAFKEY_INVALID ((uintptr_t)1)
+
+typedef struct rtree_node_elm_s rtree_node_elm_t;
+struct rtree_node_elm_s {
+	atomic_p_t	child; /* (rtree_{node,leaf}_elm_t *) */
+};
+
+struct rtree_leaf_elm_s {
+#ifdef RTREE_LEAF_COMPACT
+	/*
+	 * Single pointer-width field containing all three leaf element fields.
+	 * For example, on a 64-bit x64 system with 48 significant virtual
+	 * memory address bits, the index, extent, and slab fields are packed as
+	 * such:
+	 *
+	 * x: index
+	 * e: extent
+	 * b: slab
+	 *
+	 *   00000000 xxxxxxxx eeeeeeee [...] eeeeeeee eeee000b
+	 */
+	atomic_p_t	le_bits;
+#else
+	atomic_p_t	le_extent; /* (extent_t *) */
+	atomic_u_t	le_szind; /* (szind_t) */
+	atomic_b_t	le_slab; /* (bool) */
+#endif
+};
+
+typedef struct rtree_level_s rtree_level_t;
+struct rtree_level_s {
+	/* Number of key bits distinguished by this level. */
+	unsigned		bits;
+	/*
+	 * Cumulative number of key bits distinguished by traversing to
+	 * corresponding tree level.
+	 */
+	unsigned		cumbits;
+};
+
+typedef struct rtree_s rtree_t;
+struct rtree_s {
+	malloc_mutex_t		init_lock;
+	/* Number of elements based on rtree_levels[0].bits. */
+#if RTREE_HEIGHT > 1
+	rtree_node_elm_t	root[1U << (RTREE_NSB/RTREE_HEIGHT)];
+#else
+	rtree_leaf_elm_t	root[1U << (RTREE_NSB/RTREE_HEIGHT)];
+#endif
+};
+
+/*
+ * Split the bits into one to three partitions depending on number of
+ * significant bits.  It the number of bits does not divide evenly into the
+ * number of levels, place one remainder bit per level starting at the leaf
+ * level.
+ */
+static const rtree_level_t rtree_levels[] = {
+#if RTREE_HEIGHT == 1
+	{RTREE_NSB, RTREE_NHIB + RTREE_NSB}
+#elif RTREE_HEIGHT == 2
+	{RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2},
+	{RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB}
+#elif RTREE_HEIGHT == 3
+	{RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3},
+	{RTREE_NSB/3 + RTREE_NSB%3/2,
+	    RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2},
+	{RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB}
+#else
+#  error Unsupported rtree height
+#endif
+};
+
+bool rtree_new(rtree_t *rtree, bool zeroed);
+
+typedef rtree_node_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t);
+extern rtree_node_alloc_t *JET_MUTABLE rtree_node_alloc;
+
+typedef rtree_leaf_elm_t *(rtree_leaf_alloc_t)(tsdn_t *, rtree_t *, size_t);
+extern rtree_leaf_alloc_t *JET_MUTABLE rtree_leaf_alloc;
+
+typedef void (rtree_node_dalloc_t)(tsdn_t *, rtree_t *, rtree_node_elm_t *);
+extern rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc;
+
+typedef void (rtree_leaf_dalloc_t)(tsdn_t *, rtree_t *, rtree_leaf_elm_t *);
+extern rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc;
+#ifdef JEMALLOC_JET
+void rtree_delete(tsdn_t *tsdn, rtree_t *rtree);
+#endif
+rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,
+    rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
 
 JEMALLOC_ALWAYS_INLINE uintptr_t
 rtree_leafkey(uintptr_t key) {
@@ -347,4 +471,4 @@ rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	rtree_leaf_elm_write(tsdn, rtree, elm, NULL, NSIZES, false);
 }
 
-#endif /* JEMALLOC_INTERNAL_RTREE_INLINES_H */
+#endif /* JEMALLOC_INTERNAL_RTREE_H */

include/jemalloc/internal/rtree_ctx.h

@@ -1,22 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_RTREE_CTX_H
-#define JEMALLOC_INTERNAL_RTREE_CTX_H
-
-#include "jemalloc/internal/rtree_types.h"
-
-typedef struct rtree_ctx_cache_elm_s rtree_ctx_cache_elm_t;
-struct rtree_ctx_cache_elm_s {
-	uintptr_t		leafkey;
-	rtree_leaf_elm_t	*leaf;
-};
-
-typedef struct rtree_ctx_s rtree_ctx_t;
-struct rtree_ctx_s {
-	/* Direct mapped cache. */
-	rtree_ctx_cache_elm_t	cache[RTREE_CTX_NCACHE];
-	/* L2 LRU cache. */
-	rtree_ctx_cache_elm_t	l2_cache[RTREE_CTX_NCACHE_L2];
-};
-
-void rtree_ctx_data_init(rtree_ctx_t *ctx);
-
-#endif /* JEMALLOC_INTERNAL_RTREE_CTX_H */

include/jemalloc/internal/rtree_externs.h

@@ -1,45 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_RTREE_EXTERNS_H
-#define JEMALLOC_INTERNAL_RTREE_EXTERNS_H
-
-/*
- * Split the bits into one to three partitions depending on number of
- * significant bits.  It the number of bits does not divide evenly into the
- * number of levels, place one remainder bit per level starting at the leaf
- * level.
- */
-static const rtree_level_t rtree_levels[] = {
-#if RTREE_HEIGHT == 1
-	{RTREE_NSB, RTREE_NHIB + RTREE_NSB}
-#elif RTREE_HEIGHT == 2
-	{RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2},
-	{RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB}
-#elif RTREE_HEIGHT == 3
-	{RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3},
-	{RTREE_NSB/3 + RTREE_NSB%3/2,
-	    RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2},
-	{RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB}
-#else
-#  error Unsupported rtree height
-#endif
-};
-
-bool rtree_new(rtree_t *rtree, bool zeroed);
-
-typedef rtree_node_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t);
-extern rtree_node_alloc_t *JET_MUTABLE rtree_node_alloc;
-
-typedef rtree_leaf_elm_t *(rtree_leaf_alloc_t)(tsdn_t *, rtree_t *, size_t);
-extern rtree_leaf_alloc_t *JET_MUTABLE rtree_leaf_alloc;
-
-typedef void (rtree_node_dalloc_t)(tsdn_t *, rtree_t *, rtree_node_elm_t *);
-extern rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc;
-
-typedef void (rtree_leaf_dalloc_t)(tsdn_t *, rtree_t *, rtree_leaf_elm_t *);
-extern rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc;
-#ifdef JEMALLOC_JET
-void rtree_delete(tsdn_t *tsdn, rtree_t *rtree);
-#endif
-rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,
-    rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
-
-#endif /* JEMALLOC_INTERNAL_RTREE_EXTERNS_H */

include/jemalloc/internal/rtree_structs.h

@@ -1,53 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_RTREE_STRUCTS_H
-#define JEMALLOC_INTERNAL_RTREE_STRUCTS_H
-
-#include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/mutex.h"
-
-struct rtree_node_elm_s {
-	atomic_p_t	child; /* (rtree_{node,leaf}_elm_t *) */
-};
-
-struct rtree_leaf_elm_s {
-#ifdef RTREE_LEAF_COMPACT
-	/*
-	 * Single pointer-width field containing all three leaf element fields.
-	 * For example, on a 64-bit x64 system with 48 significant virtual
-	 * memory address bits, the index, extent, and slab fields are packed as
-	 * such:
-	 *
-	 * x: index
-	 * e: extent
-	 * b: slab
-	 *
-	 *   00000000 xxxxxxxx eeeeeeee [...] eeeeeeee eeee000b
-	 */
-	atomic_p_t	le_bits;
-#else
-	atomic_p_t	le_extent; /* (extent_t *) */
-	atomic_u_t	le_szind; /* (szind_t) */
-	atomic_b_t	le_slab; /* (bool) */
-#endif
-};
-
-struct rtree_level_s {
-	/* Number of key bits distinguished by this level. */
-	unsigned		bits;
-	/*
-	 * Cumulative number of key bits distinguished by traversing to
-	 * corresponding tree level.
-	 */
-	unsigned		cumbits;
-};
-
-struct rtree_s {
-	malloc_mutex_t		init_lock;
-	/* Number of elements based on rtree_levels[0].bits. */
-#if RTREE_HEIGHT > 1
-	rtree_node_elm_t	root[1U << (RTREE_NSB/RTREE_HEIGHT)];
-#else
-	rtree_leaf_elm_t	root[1U << (RTREE_NSB/RTREE_HEIGHT)];
-#endif
-};
-
-#endif /* JEMALLOC_INTERNAL_RTREE_STRUCTS_H */

include/jemalloc/internal/rtree_tsd.h

@@ -1,43 +1,5 @@
-#ifndef JEMALLOC_INTERNAL_RTREE_TYPES_H
+#ifndef JEMALLOC_INTERNAL_RTREE_CTX_H
-#define JEMALLOC_INTERNAL_RTREE_TYPES_H
+#define JEMALLOC_INTERNAL_RTREE_CTX_H
-
-#include "jemalloc/internal/size_classes.h"
-
-/*
- * This radix tree implementation is tailored to the singular purpose of
- * associating metadata with extents that are currently owned by jemalloc.
- *
- *******************************************************************************
- */
-
-typedef struct rtree_node_elm_s rtree_node_elm_t;
-typedef struct rtree_leaf_elm_s rtree_leaf_elm_t;
-typedef struct rtree_level_s rtree_level_t;
-typedef struct rtree_s rtree_t;
-
-/* Number of high insignificant bits. */
-#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR)
-/* Number of low insigificant bits. */
-#define RTREE_NLIB LG_PAGE
-/* Number of significant bits. */
-#define RTREE_NSB (LG_VADDR - RTREE_NLIB)
-/* Number of levels in radix tree. */
-#if RTREE_NSB <= 10
-#  define RTREE_HEIGHT 1
-#elif RTREE_NSB <= 36
-#  define RTREE_HEIGHT 2
-#elif RTREE_NSB <= 52
-#  define RTREE_HEIGHT 3
-#else
-#  error Unsupported number of significant virtual address bits
-#endif
-/* Use compact leaf representation if virtual address encoding allows. */
-#if RTREE_NHIB >= LG_CEIL_NSIZES
-#  define RTREE_LEAF_COMPACT
-#endif
-
-/* Needed for initialization only. */
-#define RTREE_LEAFKEY_INVALID ((uintptr_t)1)
 
 /*
  * Number of leafkey/leaf pairs to cache in L1 and L2 level respectively.  Each
@@ -66,4 +28,23 @@ typedef struct rtree_s rtree_t;
  */
 #define RTREE_CTX_ZERO_INITIALIZER {{{0}}}
 
-#endif /* JEMALLOC_INTERNAL_RTREE_TYPES_H */
+
+typedef struct rtree_leaf_elm_s rtree_leaf_elm_t;
+
+typedef struct rtree_ctx_cache_elm_s rtree_ctx_cache_elm_t;
+struct rtree_ctx_cache_elm_s {
+	uintptr_t		leafkey;
+	rtree_leaf_elm_t	*leaf;
+};
+
+typedef struct rtree_ctx_s rtree_ctx_t;
+struct rtree_ctx_s {
+	/* Direct mapped cache. */
+	rtree_ctx_cache_elm_t	cache[RTREE_CTX_NCACHE];
+	/* L2 LRU cache. */
+	rtree_ctx_cache_elm_t	l2_cache[RTREE_CTX_NCACHE_L2];
+};
+
+void rtree_ctx_data_init(rtree_ctx_t *ctx);
+
+#endif /* JEMALLOC_INTERNAL_RTREE_CTX_H */

include/jemalloc/internal/tsd.h

@@ -6,7 +6,7 @@
 #include "jemalloc/internal/jemalloc_internal_externs.h"
 #include "jemalloc/internal/prof_types.h"
 #include "jemalloc/internal/ql.h"
-#include "jemalloc/internal/rtree_ctx.h"
+#include "jemalloc/internal/rtree_tsd.h"
 #include "jemalloc/internal/tcache_types.h"
 #include "jemalloc/internal/tcache_structs.h"
 #include "jemalloc/internal/util.h"

src/arena.c

@@ -4,6 +4,7 @@
 
 #include "jemalloc/internal/assert.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/rtree.h"
 #include "jemalloc/internal/size_classes.h"
 #include "jemalloc/internal/util.h"
 

src/extent.c

@@ -4,6 +4,7 @@
 
 #include "jemalloc/internal/assert.h"
 #include "jemalloc/internal/ph.h"
+#include "jemalloc/internal/rtree.h"
 #include "jemalloc/internal/mutex.h"
 
 /******************************************************************************/

src/jemalloc.c

@@ -8,6 +8,7 @@
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/rtree.h"
 #include "jemalloc/internal/size_classes.h"
 #include "jemalloc/internal/spin.h"
 #include "jemalloc/internal/ticker.h"

src/large.c

@@ -4,6 +4,7 @@
 
 #include "jemalloc/internal/assert.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/rtree.h"
 #include "jemalloc/internal/util.h"
 
 /******************************************************************************/

src/tsd.c

@@ -4,6 +4,7 @@
 
 #include "jemalloc/internal/assert.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/rtree.h"
 
 /******************************************************************************/
 /* Data. */

test/unit/arena_reset.c

@@ -2,6 +2,8 @@
 #include "test/jemalloc_test.h"
 #endif
 
+#include "jemalloc/internal/rtree.h"
+
 #include "test/extent_hooks.h"
 
 static unsigned

test/unit/rtree.c

@@ -1,5 +1,7 @@
 #include "test/jemalloc_test.h"
 
+#include "jemalloc/internal/rtree.h"
+
 rtree_node_alloc_t *rtree_node_alloc_orig;
 rtree_node_dalloc_t *rtree_node_dalloc_orig;
 rtree_leaf_alloc_t *rtree_leaf_alloc_orig;

test/unit/spin.c

@@ -1,5 +1,7 @@
 #include "test/jemalloc_test.h"
 
+#include "jemalloc/internal/spin.h"
+
 TEST_BEGIN(test_spin) {
 	spin_t spinner = SPIN_INITIALIZER;