From ce41ab0c57d5f0c9310200d0fecea99ef334a834 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 22 Mar 2017 16:38:03 -0700 Subject: [PATCH] Embed root node into rtree_t. This avoids one atomic operation per tree access. --- include/jemalloc/internal/rtree_externs.h | 2 +- include/jemalloc/internal/rtree_structs.h | 12 ++- src/extent.c | 2 +- src/rtree.c | 91 ++++------------- test/unit/rtree.c | 119 ++++++++++------------ 5 files changed, 86 insertions(+), 140 deletions(-) diff --git a/include/jemalloc/internal/rtree_externs.h b/include/jemalloc/internal/rtree_externs.h index 842eb0b5..5145c12c 100644 --- a/include/jemalloc/internal/rtree_externs.h +++ b/include/jemalloc/internal/rtree_externs.h @@ -23,7 +23,7 @@ static const rtree_level_t rtree_levels[] = { #endif }; -bool rtree_new(rtree_t *rtree); +bool rtree_new(rtree_t *rtree, bool zeroed); #ifdef JEMALLOC_JET typedef rtree_node_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t); extern rtree_node_alloc_t *rtree_node_alloc; diff --git a/include/jemalloc/internal/rtree_structs.h b/include/jemalloc/internal/rtree_structs.h index e9a507ab..3ecdf810 100644 --- a/include/jemalloc/internal/rtree_structs.h +++ b/include/jemalloc/internal/rtree_structs.h @@ -39,13 +39,17 @@ struct rtree_ctx_s { #ifndef _MSC_VER JEMALLOC_ALIGNED(CACHELINE) #endif - rtree_ctx_cache_elm_t cache[RTREE_CTX_NCACHE]; + rtree_ctx_cache_elm_t cache[RTREE_CTX_NCACHE]; }; struct rtree_s { - /* An rtree_{internal,leaf}_elm_t *. */ - atomic_p_t root; - malloc_mutex_t init_lock; + malloc_mutex_t init_lock; + /* Number of elements based on rtree_levels[0].bits. */ +#if RTREE_HEIGHT > 1 + rtree_node_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; +#else + rtree_leaf_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; +#endif }; #endif /* JEMALLOC_INTERNAL_RTREE_STRUCTS_H */ diff --git a/src/extent.c b/src/extent.c index a95e2748..f1b513e4 100644 --- a/src/extent.c +++ b/src/extent.c @@ -1658,7 +1658,7 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_boot(void) { - if (rtree_new(&extents_rtree)) { + if (rtree_new(&extents_rtree, true)) { return true; } diff --git a/src/rtree.c b/src/rtree.c index 18197390..a07380f3 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -6,8 +6,15 @@ * used. */ bool -rtree_new(rtree_t *rtree) { - atomic_store_p(&rtree->root, NULL, ATOMIC_RELAXED); +rtree_new(rtree_t *rtree, bool zeroed) { +#ifdef JEMALLOC_JET + if (!zeroed) { + memset(rtree, 0, sizeof(rtree_t)); /* Clear root. */ + } +#else + assert(zeroed); +#endif + if (malloc_mutex_init(&rtree->init_lock, "rtree", WITNESS_RANK_RTREE)) { return true; } @@ -76,6 +83,7 @@ rtree_leaf_dalloc_t *rtree_leaf_dalloc = JEMALLOC_N(rtree_leaf_dalloc_impl); #endif #ifdef JEMALLOC_JET +# if RTREE_HEIGHT > 1 static void rtree_delete_subtree(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *subtree, unsigned level) { @@ -101,25 +109,17 @@ rtree_delete_subtree(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *subtree, } } - rtree_node_dalloc(tsdn, rtree, subtree); + if (subtree != rtree->root) { + rtree_node_dalloc(tsdn, rtree, subtree); + } } +# endif void rtree_delete(tsdn_t *tsdn, rtree_t *rtree) { - if (RTREE_HEIGHT > 1) { - rtree_node_elm_t *node = (rtree_node_elm_t *)atomic_load_p( - &rtree->root, ATOMIC_RELAXED); - if (node != NULL) { - rtree_delete_subtree(tsdn, rtree, node, 0); - } - } else { - rtree_leaf_elm_t *leaf = - (rtree_leaf_elm_t *)atomic_load_p(&rtree->root, - ATOMIC_RELAXED); - if (leaf != NULL) { - rtree_leaf_dalloc(tsdn, rtree, leaf); - } - } +# if RTREE_HEIGHT > 1 + rtree_delete_subtree(tsdn, rtree, rtree->root, 0); +# endif } #endif @@ -244,70 +244,21 @@ rtree_child_leaf_read(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *elm, return leaf; } -UNUSED static rtree_node_elm_t * -rtree_root_node_tryread(rtree_t *rtree, bool dependent) { - rtree_node_elm_t *node; - if (dependent) { - node = (rtree_node_elm_t *)atomic_load_p(&rtree->root, - ATOMIC_RELAXED); - } else { - node = (rtree_node_elm_t *)atomic_load_p(&rtree->root, - ATOMIC_ACQUIRE); - } - assert(!dependent || node != NULL); - return node; -} - -UNUSED static rtree_node_elm_t * -rtree_root_node_read(tsdn_t *tsdn, rtree_t *rtree, bool dependent) { - rtree_node_elm_t *node = rtree_root_node_tryread(rtree, dependent); - if (!dependent && unlikely(!rtree_node_valid(node))) { - node = rtree_node_init(tsdn, rtree, 0, &rtree->root); - } - assert(!dependent || node != NULL); - return node; -} - -UNUSED static rtree_leaf_elm_t * -rtree_root_leaf_tryread(rtree_t *rtree, bool dependent) { - rtree_leaf_elm_t *leaf; - if (dependent) { - leaf = (rtree_leaf_elm_t *)atomic_load_p(&rtree->root, - ATOMIC_RELAXED); - } else { - leaf = (rtree_leaf_elm_t *)atomic_load_p(&rtree->root, - ATOMIC_ACQUIRE); - } - assert(!dependent || leaf != NULL); - return leaf; -} - -UNUSED static rtree_leaf_elm_t * -rtree_root_leaf_read(tsdn_t *tsdn, rtree_t *rtree, bool dependent) { - rtree_leaf_elm_t *leaf = rtree_root_leaf_tryread(rtree, dependent); - if (!dependent && unlikely(!rtree_leaf_valid(leaf))) { - leaf = rtree_leaf_init(tsdn, rtree, &rtree->root); - } - assert(!dependent || leaf != NULL); - return leaf; -} - rtree_leaf_elm_t * rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing) { rtree_node_elm_t *node; rtree_leaf_elm_t *leaf; #if RTREE_HEIGHT > 1 - node = init_missing ? rtree_root_node_read(tsdn, rtree, dependent) : - rtree_root_node_tryread(rtree, dependent); + node = rtree->root; #else - leaf = init_missing ? rtree_root_leaf_read(tsdn, rtree, dependent) : - rtree_root_leaf_tryread(rtree, dependent); + leaf = rtree->root; #endif #define RTREE_GET_CHILD(level) { \ assert(level < RTREE_HEIGHT-1); \ - if (!dependent && unlikely(!rtree_node_valid(node))) { \ + if (level != 0 && !dependent && \ + unlikely(!rtree_node_valid(node))) { \ return NULL; \ } \ uintptr_t subkey = rtree_subkey(key, level); \ diff --git a/test/unit/rtree.c b/test/unit/rtree.c index b04e321b..7a25c47d 100644 --- a/test/unit/rtree.c +++ b/test/unit/rtree.c @@ -5,13 +5,14 @@ rtree_node_dalloc_t *rtree_node_dalloc_orig; rtree_leaf_alloc_t *rtree_leaf_alloc_orig; rtree_leaf_dalloc_t *rtree_leaf_dalloc_orig; -rtree_t *test_rtree; +/* Potentially too large to safely place on the stack. */ +rtree_t test_rtree; static rtree_node_elm_t * rtree_node_alloc_intercept(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) { rtree_node_elm_t *node; - if (rtree != test_rtree) { + if (rtree != &test_rtree) { return rtree_node_alloc_orig(tsdn, rtree, nelms); } @@ -26,7 +27,7 @@ rtree_node_alloc_intercept(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) { static void rtree_node_dalloc_intercept(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) { - if (rtree != test_rtree) { + if (rtree != &test_rtree) { rtree_node_dalloc_orig(tsdn, rtree, node); return; } @@ -38,7 +39,7 @@ static rtree_leaf_elm_t * rtree_leaf_alloc_intercept(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) { rtree_leaf_elm_t *leaf; - if (rtree != test_rtree) { + if (rtree != &test_rtree) { return rtree_leaf_alloc_orig(tsdn, rtree, nelms); } @@ -53,7 +54,7 @@ rtree_leaf_alloc_intercept(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) { static void rtree_leaf_dalloc_intercept(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) { - if (rtree != test_rtree) { + if (rtree != &test_rtree) { rtree_leaf_dalloc_orig(tsdn, rtree, leaf); return; } @@ -66,14 +67,12 @@ TEST_BEGIN(test_rtree_read_empty) { tsdn = tsdn_fetch(); - rtree_t rtree; + rtree_t *rtree = &test_rtree; rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; - test_rtree = &rtree; - assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure"); - assert_ptr_null(rtree_extent_read(tsdn, &rtree, &rtree_ctx, PAGE, + assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); + assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx, PAGE, false), "rtree_extent_read() should return NULL for empty tree"); - rtree_delete(tsdn, &rtree); - test_rtree = NULL; + rtree_delete(tsdn, rtree); } TEST_END @@ -83,7 +82,7 @@ TEST_END #define SEED 42 typedef struct { - rtree_t rtree; + rtree_t *rtree; uint32_t seed; } thd_start_arg_t; @@ -108,26 +107,26 @@ thd_start(void *varg) { MAX_NBITS) - ZU(1))); if (i % 2 == 0) { rtree_leaf_elm_t *elm = rtree_leaf_elm_acquire(tsdn, - &arg->rtree, &rtree_ctx, key, false, true); + arg->rtree, &rtree_ctx, key, false, true); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_acquire() failure"); - rtree_leaf_elm_write(tsdn, &arg->rtree, elm, true, + rtree_leaf_elm_write(tsdn, arg->rtree, elm, true, extent, NSIZES, false); - rtree_leaf_elm_release(tsdn, &arg->rtree, elm); + rtree_leaf_elm_release(tsdn, arg->rtree, elm); - elm = rtree_leaf_elm_acquire(tsdn, &arg->rtree, + elm = rtree_leaf_elm_acquire(tsdn, arg->rtree, &rtree_ctx, key, true, false); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_acquire() failure"); - rtree_leaf_elm_extent_read(tsdn, &arg->rtree, elm, true, + rtree_leaf_elm_extent_read(tsdn, arg->rtree, elm, true, true); - rtree_leaf_elm_szind_read(tsdn, &arg->rtree, elm, true, + rtree_leaf_elm_szind_read(tsdn, arg->rtree, elm, true, true); - rtree_leaf_elm_slab_read(tsdn, &arg->rtree, elm, true, + rtree_leaf_elm_slab_read(tsdn, arg->rtree, elm, true, true); - rtree_leaf_elm_release(tsdn, &arg->rtree, elm); + rtree_leaf_elm_release(tsdn, arg->rtree, elm); } else { - rtree_extent_read(tsdn, &arg->rtree, &rtree_ctx, key, + rtree_extent_read(tsdn, arg->rtree, &rtree_ctx, key, false); } } @@ -145,8 +144,9 @@ TEST_BEGIN(test_rtree_concurrent) { sfmt = init_gen_rand(SEED); tsdn = tsdn_fetch(); - test_rtree = &arg.rtree; - assert_false(rtree_new(&arg.rtree), "Unexpected rtree_new() failure"); + arg.rtree = &test_rtree; + assert_false(rtree_new(arg.rtree, false), + "Unexpected rtree_new() failure"); arg.seed = gen_rand32(sfmt); for (unsigned i = 0; i < NTHREADS; i++) { thd_create(&thds[i], thd_start, (void *)&arg); @@ -154,8 +154,7 @@ TEST_BEGIN(test_rtree_concurrent) { for (unsigned i = 0; i < NTHREADS; i++) { thd_join(thds[i], NULL); } - rtree_delete(tsdn, &arg.rtree); - test_rtree = NULL; + rtree_delete(tsdn, arg.rtree); fini_gen_rand(sfmt); } TEST_END @@ -173,29 +172,27 @@ TEST_BEGIN(test_rtree_extrema) { tsdn_t *tsdn = tsdn_fetch(); - rtree_t rtree; + rtree_t *rtree = &test_rtree; rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; - test_rtree = &rtree; - assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure"); + assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); - assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, PAGE, &extent_a, + assert_false(rtree_write(tsdn, rtree, &rtree_ctx, PAGE, &extent_a, extent_szind_get(&extent_a), extent_slab_get(&extent_a)), "Unexpected rtree_write() failure"); - rtree_szind_slab_update(tsdn, &rtree, &rtree_ctx, PAGE, + rtree_szind_slab_update(tsdn, rtree, &rtree_ctx, PAGE, extent_szind_get(&extent_a), extent_slab_get(&extent_a)); - assert_ptr_eq(rtree_extent_read(tsdn, &rtree, &rtree_ctx, PAGE, true), + assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, PAGE, true), &extent_a, "rtree_extent_read() should return previously set value"); - assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, ~((uintptr_t)0), + assert_false(rtree_write(tsdn, rtree, &rtree_ctx, ~((uintptr_t)0), &extent_b, extent_szind_get_maybe_invalid(&extent_b), extent_slab_get(&extent_b)), "Unexpected rtree_write() failure"); - assert_ptr_eq(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, ~((uintptr_t)0), true), &extent_b, "rtree_extent_read() should return previously set value"); - rtree_delete(tsdn, &rtree); - test_rtree = NULL; + rtree_delete(tsdn, rtree); } TEST_END @@ -209,33 +206,30 @@ TEST_BEGIN(test_rtree_bits) { extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, extent_state_active, false, false); - rtree_t rtree; + rtree_t *rtree = &test_rtree; rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; - test_rtree = &rtree; - assert_false(rtree_new(&rtree), - "Unexpected rtree_new() failure"); + assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) { - assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, keys[i], + assert_false(rtree_write(tsdn, rtree, &rtree_ctx, keys[i], &extent, NSIZES, false), "Unexpected rtree_write() failure"); for (unsigned j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { - assert_ptr_eq(rtree_extent_read(tsdn, &rtree, - &rtree_ctx, keys[j], true), - &extent, "rtree_extent_read() should return " - "previously set value and ignore insignificant key " - "bits; i=%u, j=%u, set key=%#"FMTxPTR", get " - "key=%#"FMTxPTR, i, j, keys[i], keys[j]); + assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, + keys[j], true), &extent, + "rtree_extent_read() should return previously set " + "value and ignore insignificant key bits; i=%u, " + "j=%u, set key=%#"FMTxPTR", get key=%#"FMTxPTR, i, + j, keys[i], keys[j]); } - assert_ptr_null(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx, (((uintptr_t)2) << LG_PAGE), false), "Only leftmost rtree leaf should be set; i=%u", i); - rtree_clear(tsdn, &rtree, &rtree_ctx, keys[i]); + rtree_clear(tsdn, rtree, &rtree_ctx, keys[i]); } - rtree_delete(tsdn, &rtree); - test_rtree = NULL; + rtree_delete(tsdn, rtree); } TEST_END @@ -245,50 +239,48 @@ TEST_BEGIN(test_rtree_random) { sfmt_t *sfmt = init_gen_rand(SEED); tsdn_t *tsdn = tsdn_fetch(); uintptr_t keys[NSET]; - rtree_t rtree; + rtree_t *rtree = &test_rtree; rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; extent_t extent; extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, extent_state_active, false, false); - test_rtree = &rtree; - assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure"); + assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); for (unsigned i = 0; i < NSET; i++) { keys[i] = (uintptr_t)gen_rand64(sfmt); - rtree_leaf_elm_t *elm = rtree_leaf_elm_acquire(tsdn, &rtree, + rtree_leaf_elm_t *elm = rtree_leaf_elm_acquire(tsdn, rtree, &rtree_ctx, keys[i], false, true); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_acquire() failure"); - rtree_leaf_elm_write(tsdn, &rtree, elm, true, &extent, NSIZES, + rtree_leaf_elm_write(tsdn, rtree, elm, true, &extent, NSIZES, false); - rtree_leaf_elm_release(tsdn, &rtree, elm); - assert_ptr_eq(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + rtree_leaf_elm_release(tsdn, rtree, elm); + assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, keys[i], true), &extent, "rtree_extent_read() should return previously set value"); } for (unsigned i = 0; i < NSET; i++) { - assert_ptr_eq(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, keys[i], true), &extent, "rtree_extent_read() should return previously set value, " "i=%u", i); } for (unsigned i = 0; i < NSET; i++) { - rtree_clear(tsdn, &rtree, &rtree_ctx, keys[i]); - assert_ptr_null(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + rtree_clear(tsdn, rtree, &rtree_ctx, keys[i]); + assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx, keys[i], true), "rtree_extent_read() should return previously set value"); } for (unsigned i = 0; i < NSET; i++) { - assert_ptr_null(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx, keys[i], true), "rtree_extent_read() should return previously set value"); } - rtree_delete(tsdn, &rtree); - test_rtree = NULL; + rtree_delete(tsdn, rtree); fini_gen_rand(sfmt); #undef NSET #undef SEED @@ -305,7 +297,6 @@ main(void) { rtree_leaf_alloc = rtree_leaf_alloc_intercept; rtree_leaf_dalloc_orig = rtree_leaf_dalloc; rtree_leaf_dalloc = rtree_leaf_dalloc_intercept; - test_rtree = NULL; return test( test_rtree_read_empty,