diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 8db6e9a8..16635c1a 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -3,6 +3,7 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/ticker.h" diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent_externs.h index 9d5daf5b..96a71126 100644 --- a/include/jemalloc/internal/extent_externs.h +++ b/include/jemalloc/internal/extent_externs.h @@ -4,6 +4,7 @@ #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/ph.h" #include "jemalloc/internal/rb.h" +#include "jemalloc/internal/rtree.h" extern rtree_t extents_rtree; extern const extent_hooks_t extent_hooks_default; diff --git a/include/jemalloc/internal/jemalloc_internal_includes.h b/include/jemalloc/internal/jemalloc_internal_includes.h index b1a6f17d..770bcaab 100644 --- a/include/jemalloc/internal/jemalloc_internal_includes.h +++ b/include/jemalloc/internal/jemalloc_internal_includes.h @@ -44,7 +44,6 @@ #include "jemalloc/internal/extent_dss_types.h" #include "jemalloc/internal/base_types.h" #include "jemalloc/internal/arena_types.h" -#include "jemalloc/internal/rtree_types.h" #include "jemalloc/internal/tcache_types.h" #include "jemalloc/internal/prof_types.h" @@ -59,7 +58,6 @@ #include "jemalloc/internal/base_structs.h" #include "jemalloc/internal/prof_structs.h" #include "jemalloc/internal/arena_structs_b.h" -#include "jemalloc/internal/rtree_structs.h" #include "jemalloc/internal/tcache_structs.h" #include "jemalloc/internal/background_thread_structs.h" @@ -73,7 +71,6 @@ #include "jemalloc/internal/extent_mmap_externs.h" #include "jemalloc/internal/base_externs.h" #include "jemalloc/internal/arena_externs.h" -#include "jemalloc/internal/rtree_externs.h" #include "jemalloc/internal/large_externs.h" #include "jemalloc/internal/tcache_externs.h" #include "jemalloc/internal/prof_externs.h" @@ -85,7 +82,6 @@ #include "jemalloc/internal/mutex_pool_inlines.h" #include "jemalloc/internal/jemalloc_internal_inlines_a.h" -#include "jemalloc/internal/rtree_inlines.h" #include "jemalloc/internal/base_inlines.h" /* * Include portions of arena code interleaved with tcache code in order to diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/include/jemalloc/internal/jemalloc_internal_inlines_b.h index cfc52094..37493160 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_b.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_INLINES_B_H #define JEMALLOC_INTERNAL_INLINES_B_H +#include "jemalloc/internal/rtree.h" + /* Choose an arena based on a per-thread value. */ static inline arena_t * arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { diff --git a/include/jemalloc/internal/rtree_inlines.h b/include/jemalloc/internal/rtree.h similarity index 74% rename from include/jemalloc/internal/rtree_inlines.h rename to include/jemalloc/internal/rtree.h index 335a89cf..b5d4db39 100644 --- a/include/jemalloc/internal/rtree_inlines.h +++ b/include/jemalloc/internal/rtree.h @@ -1,8 +1,132 @@ -#ifndef JEMALLOC_INTERNAL_RTREE_INLINES_H -#define JEMALLOC_INTERNAL_RTREE_INLINES_H +#ifndef JEMALLOC_INTERNAL_RTREE_H +#define JEMALLOC_INTERNAL_RTREE_H +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/rtree_tsd.h" #include "jemalloc/internal/size_classes.h" -#include "jemalloc/internal/spin.h" +#include "jemalloc/internal/tsd.h" + +/* + * This radix tree implementation is tailored to the singular purpose of + * associating metadata with extents that are currently owned by jemalloc. + * + ******************************************************************************* + */ + +/* Number of high insignificant bits. */ +#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR) +/* Number of low insigificant bits. */ +#define RTREE_NLIB LG_PAGE +/* Number of significant bits. */ +#define RTREE_NSB (LG_VADDR - RTREE_NLIB) +/* Number of levels in radix tree. */ +#if RTREE_NSB <= 10 +# define RTREE_HEIGHT 1 +#elif RTREE_NSB <= 36 +# define RTREE_HEIGHT 2 +#elif RTREE_NSB <= 52 +# define RTREE_HEIGHT 3 +#else +# error Unsupported number of significant virtual address bits +#endif +/* Use compact leaf representation if virtual address encoding allows. */ +#if RTREE_NHIB >= LG_CEIL_NSIZES +# define RTREE_LEAF_COMPACT +#endif + +/* Needed for initialization only. */ +#define RTREE_LEAFKEY_INVALID ((uintptr_t)1) + +typedef struct rtree_node_elm_s rtree_node_elm_t; +struct rtree_node_elm_s { + atomic_p_t child; /* (rtree_{node,leaf}_elm_t *) */ +}; + +struct rtree_leaf_elm_s { +#ifdef RTREE_LEAF_COMPACT + /* + * Single pointer-width field containing all three leaf element fields. + * For example, on a 64-bit x64 system with 48 significant virtual + * memory address bits, the index, extent, and slab fields are packed as + * such: + * + * x: index + * e: extent + * b: slab + * + * 00000000 xxxxxxxx eeeeeeee [...] eeeeeeee eeee000b + */ + atomic_p_t le_bits; +#else + atomic_p_t le_extent; /* (extent_t *) */ + atomic_u_t le_szind; /* (szind_t) */ + atomic_b_t le_slab; /* (bool) */ +#endif +}; + +typedef struct rtree_level_s rtree_level_t; +struct rtree_level_s { + /* Number of key bits distinguished by this level. */ + unsigned bits; + /* + * Cumulative number of key bits distinguished by traversing to + * corresponding tree level. + */ + unsigned cumbits; +}; + +typedef struct rtree_s rtree_t; +struct rtree_s { + malloc_mutex_t init_lock; + /* Number of elements based on rtree_levels[0].bits. */ +#if RTREE_HEIGHT > 1 + rtree_node_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; +#else + rtree_leaf_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; +#endif +}; + +/* + * Split the bits into one to three partitions depending on number of + * significant bits. It the number of bits does not divide evenly into the + * number of levels, place one remainder bit per level starting at the leaf + * level. + */ +static const rtree_level_t rtree_levels[] = { +#if RTREE_HEIGHT == 1 + {RTREE_NSB, RTREE_NHIB + RTREE_NSB} +#elif RTREE_HEIGHT == 2 + {RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2}, + {RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB} +#elif RTREE_HEIGHT == 3 + {RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3}, + {RTREE_NSB/3 + RTREE_NSB%3/2, + RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2}, + {RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB} +#else +# error Unsupported rtree height +#endif +}; + +bool rtree_new(rtree_t *rtree, bool zeroed); + +typedef rtree_node_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t); +extern rtree_node_alloc_t *JET_MUTABLE rtree_node_alloc; + +typedef rtree_leaf_elm_t *(rtree_leaf_alloc_t)(tsdn_t *, rtree_t *, size_t); +extern rtree_leaf_alloc_t *JET_MUTABLE rtree_leaf_alloc; + +typedef void (rtree_node_dalloc_t)(tsdn_t *, rtree_t *, rtree_node_elm_t *); +extern rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc; + +typedef void (rtree_leaf_dalloc_t)(tsdn_t *, rtree_t *, rtree_leaf_elm_t *); +extern rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc; +#ifdef JEMALLOC_JET +void rtree_delete(tsdn_t *tsdn, rtree_t *rtree); +#endif +rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, + rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing); JEMALLOC_ALWAYS_INLINE uintptr_t rtree_leafkey(uintptr_t key) { @@ -347,4 +471,4 @@ rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_write(tsdn, rtree, elm, NULL, NSIZES, false); } -#endif /* JEMALLOC_INTERNAL_RTREE_INLINES_H */ +#endif /* JEMALLOC_INTERNAL_RTREE_H */ diff --git a/include/jemalloc/internal/rtree_ctx.h b/include/jemalloc/internal/rtree_ctx.h deleted file mode 100644 index fe2c8bde..00000000 --- a/include/jemalloc/internal/rtree_ctx.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_RTREE_CTX_H -#define JEMALLOC_INTERNAL_RTREE_CTX_H - -#include "jemalloc/internal/rtree_types.h" - -typedef struct rtree_ctx_cache_elm_s rtree_ctx_cache_elm_t; -struct rtree_ctx_cache_elm_s { - uintptr_t leafkey; - rtree_leaf_elm_t *leaf; -}; - -typedef struct rtree_ctx_s rtree_ctx_t; -struct rtree_ctx_s { - /* Direct mapped cache. */ - rtree_ctx_cache_elm_t cache[RTREE_CTX_NCACHE]; - /* L2 LRU cache. */ - rtree_ctx_cache_elm_t l2_cache[RTREE_CTX_NCACHE_L2]; -}; - -void rtree_ctx_data_init(rtree_ctx_t *ctx); - -#endif /* JEMALLOC_INTERNAL_RTREE_CTX_H */ diff --git a/include/jemalloc/internal/rtree_externs.h b/include/jemalloc/internal/rtree_externs.h deleted file mode 100644 index d7d81654..00000000 --- a/include/jemalloc/internal/rtree_externs.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_RTREE_EXTERNS_H -#define JEMALLOC_INTERNAL_RTREE_EXTERNS_H - -/* - * Split the bits into one to three partitions depending on number of - * significant bits. It the number of bits does not divide evenly into the - * number of levels, place one remainder bit per level starting at the leaf - * level. - */ -static const rtree_level_t rtree_levels[] = { -#if RTREE_HEIGHT == 1 - {RTREE_NSB, RTREE_NHIB + RTREE_NSB} -#elif RTREE_HEIGHT == 2 - {RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2}, - {RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB} -#elif RTREE_HEIGHT == 3 - {RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3}, - {RTREE_NSB/3 + RTREE_NSB%3/2, - RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2}, - {RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB} -#else -# error Unsupported rtree height -#endif -}; - -bool rtree_new(rtree_t *rtree, bool zeroed); - -typedef rtree_node_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t); -extern rtree_node_alloc_t *JET_MUTABLE rtree_node_alloc; - -typedef rtree_leaf_elm_t *(rtree_leaf_alloc_t)(tsdn_t *, rtree_t *, size_t); -extern rtree_leaf_alloc_t *JET_MUTABLE rtree_leaf_alloc; - -typedef void (rtree_node_dalloc_t)(tsdn_t *, rtree_t *, rtree_node_elm_t *); -extern rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc; - -typedef void (rtree_leaf_dalloc_t)(tsdn_t *, rtree_t *, rtree_leaf_elm_t *); -extern rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc; -#ifdef JEMALLOC_JET -void rtree_delete(tsdn_t *tsdn, rtree_t *rtree); -#endif -rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, - rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing); - -#endif /* JEMALLOC_INTERNAL_RTREE_EXTERNS_H */ diff --git a/include/jemalloc/internal/rtree_structs.h b/include/jemalloc/internal/rtree_structs.h deleted file mode 100644 index a02a1f60..00000000 --- a/include/jemalloc/internal/rtree_structs.h +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_RTREE_STRUCTS_H -#define JEMALLOC_INTERNAL_RTREE_STRUCTS_H - -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/mutex.h" - -struct rtree_node_elm_s { - atomic_p_t child; /* (rtree_{node,leaf}_elm_t *) */ -}; - -struct rtree_leaf_elm_s { -#ifdef RTREE_LEAF_COMPACT - /* - * Single pointer-width field containing all three leaf element fields. - * For example, on a 64-bit x64 system with 48 significant virtual - * memory address bits, the index, extent, and slab fields are packed as - * such: - * - * x: index - * e: extent - * b: slab - * - * 00000000 xxxxxxxx eeeeeeee [...] eeeeeeee eeee000b - */ - atomic_p_t le_bits; -#else - atomic_p_t le_extent; /* (extent_t *) */ - atomic_u_t le_szind; /* (szind_t) */ - atomic_b_t le_slab; /* (bool) */ -#endif -}; - -struct rtree_level_s { - /* Number of key bits distinguished by this level. */ - unsigned bits; - /* - * Cumulative number of key bits distinguished by traversing to - * corresponding tree level. - */ - unsigned cumbits; -}; - -struct rtree_s { - malloc_mutex_t init_lock; - /* Number of elements based on rtree_levels[0].bits. */ -#if RTREE_HEIGHT > 1 - rtree_node_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; -#else - rtree_leaf_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; -#endif -}; - -#endif /* JEMALLOC_INTERNAL_RTREE_STRUCTS_H */ diff --git a/include/jemalloc/internal/rtree_types.h b/include/jemalloc/internal/rtree_tsd.h similarity index 51% rename from include/jemalloc/internal/rtree_types.h rename to include/jemalloc/internal/rtree_tsd.h index fd0f1409..3cdc8625 100644 --- a/include/jemalloc/internal/rtree_types.h +++ b/include/jemalloc/internal/rtree_tsd.h @@ -1,43 +1,5 @@ -#ifndef JEMALLOC_INTERNAL_RTREE_TYPES_H -#define JEMALLOC_INTERNAL_RTREE_TYPES_H - -#include "jemalloc/internal/size_classes.h" - -/* - * This radix tree implementation is tailored to the singular purpose of - * associating metadata with extents that are currently owned by jemalloc. - * - ******************************************************************************* - */ - -typedef struct rtree_node_elm_s rtree_node_elm_t; -typedef struct rtree_leaf_elm_s rtree_leaf_elm_t; -typedef struct rtree_level_s rtree_level_t; -typedef struct rtree_s rtree_t; - -/* Number of high insignificant bits. */ -#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR) -/* Number of low insigificant bits. */ -#define RTREE_NLIB LG_PAGE -/* Number of significant bits. */ -#define RTREE_NSB (LG_VADDR - RTREE_NLIB) -/* Number of levels in radix tree. */ -#if RTREE_NSB <= 10 -# define RTREE_HEIGHT 1 -#elif RTREE_NSB <= 36 -# define RTREE_HEIGHT 2 -#elif RTREE_NSB <= 52 -# define RTREE_HEIGHT 3 -#else -# error Unsupported number of significant virtual address bits -#endif -/* Use compact leaf representation if virtual address encoding allows. */ -#if RTREE_NHIB >= LG_CEIL_NSIZES -# define RTREE_LEAF_COMPACT -#endif - -/* Needed for initialization only. */ -#define RTREE_LEAFKEY_INVALID ((uintptr_t)1) +#ifndef JEMALLOC_INTERNAL_RTREE_CTX_H +#define JEMALLOC_INTERNAL_RTREE_CTX_H /* * Number of leafkey/leaf pairs to cache in L1 and L2 level respectively. Each @@ -66,4 +28,23 @@ typedef struct rtree_s rtree_t; */ #define RTREE_CTX_ZERO_INITIALIZER {{{0}}} -#endif /* JEMALLOC_INTERNAL_RTREE_TYPES_H */ + +typedef struct rtree_leaf_elm_s rtree_leaf_elm_t; + +typedef struct rtree_ctx_cache_elm_s rtree_ctx_cache_elm_t; +struct rtree_ctx_cache_elm_s { + uintptr_t leafkey; + rtree_leaf_elm_t *leaf; +}; + +typedef struct rtree_ctx_s rtree_ctx_t; +struct rtree_ctx_s { + /* Direct mapped cache. */ + rtree_ctx_cache_elm_t cache[RTREE_CTX_NCACHE]; + /* L2 LRU cache. */ + rtree_ctx_cache_elm_t l2_cache[RTREE_CTX_NCACHE_L2]; +}; + +void rtree_ctx_data_init(rtree_ctx_t *ctx); + +#endif /* JEMALLOC_INTERNAL_RTREE_CTX_H */ diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index c192a6ca..f304e1d9 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -6,7 +6,7 @@ #include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/prof_types.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/rtree_ctx.h" +#include "jemalloc/internal/rtree_tsd.h" #include "jemalloc/internal/tcache_types.h" #include "jemalloc/internal/tcache_structs.h" #include "jemalloc/internal/util.h" diff --git a/src/arena.c b/src/arena.c index 9b3ea235..3d0725f4 100644 --- a/src/arena.c +++ b/src/arena.c @@ -4,6 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/util.h" diff --git a/src/extent.c b/src/extent.c index 44e98789..2264a0ca 100644 --- a/src/extent.c +++ b/src/extent.c @@ -4,6 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/ph.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/mutex.h" /******************************************************************************/ diff --git a/src/jemalloc.c b/src/jemalloc.c index ed22a258..00b645f0 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -8,6 +8,7 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/spin.h" #include "jemalloc/internal/ticker.h" diff --git a/src/large.c b/src/large.c index 55ee3524..27c9bc60 100644 --- a/src/large.c +++ b/src/large.c @@ -4,6 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/util.h" /******************************************************************************/ diff --git a/src/tsd.c b/src/tsd.c index 525432b6..29a56775 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -4,6 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/rtree.h" /******************************************************************************/ /* Data. */ diff --git a/test/unit/arena_reset.c b/test/unit/arena_reset.c index d1698325..678ae57c 100644 --- a/test/unit/arena_reset.c +++ b/test/unit/arena_reset.c @@ -2,6 +2,8 @@ #include "test/jemalloc_test.h" #endif +#include "jemalloc/internal/rtree.h" + #include "test/extent_hooks.h" static unsigned diff --git a/test/unit/rtree.c b/test/unit/rtree.c index b854afd7..752dde99 100644 --- a/test/unit/rtree.c +++ b/test/unit/rtree.c @@ -1,5 +1,7 @@ #include "test/jemalloc_test.h" +#include "jemalloc/internal/rtree.h" + rtree_node_alloc_t *rtree_node_alloc_orig; rtree_node_dalloc_t *rtree_node_dalloc_orig; rtree_leaf_alloc_t *rtree_leaf_alloc_orig; diff --git a/test/unit/spin.c b/test/unit/spin.c index bd368b3d..b965f742 100644 --- a/test/unit/spin.c +++ b/test/unit/spin.c @@ -1,5 +1,7 @@ #include "test/jemalloc_test.h" +#include "jemalloc/internal/spin.h" + TEST_BEGIN(test_spin) { spin_t spinner = SPIN_INITIALIZER;