From 99d68445efa40edaf6c5317179faea4ecd07345a Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 16 Mar 2017 17:57:52 -0700 Subject: [PATCH] Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer. --- include/jemalloc/internal/arena_inlines_b.h | 55 +++- include/jemalloc/internal/extent_externs.h | 2 +- include/jemalloc/internal/extent_inlines.h | 22 +- .../jemalloc/internal/jemalloc_internal.h.in | 28 +- include/jemalloc/internal/large_externs.h | 4 +- include/jemalloc/internal/private_symbols.txt | 17 +- include/jemalloc/internal/rtree_inlines.h | 297 ++++++++++++------ include/jemalloc/internal/rtree_structs.h | 6 +- src/arena.c | 11 +- src/extent.c | 127 +++++--- src/large.c | 17 +- test/unit/arena_reset.c | 17 +- test/unit/rtree.c | 94 ++++-- 13 files changed, 471 insertions(+), 226 deletions(-) diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 92c89a5b..8c5f9c14 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -14,7 +14,8 @@ void arena_decay_tick(tsdn_t *tsdn, arena_t *arena); void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool slow_path); arena_t *arena_aalloc(tsdn_t *tsdn, const void *ptr); -size_t arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr); +size_t arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr); +size_t arena_vsalloc(tsdn_t *tsdn, const void *ptr); void arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache, bool slow_path); void arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size, @@ -114,12 +115,60 @@ arena_aalloc(tsdn_t *tsdn, const void *ptr) { return extent_arena_get(iealloc(tsdn, ptr)); } -/* Return the size of the allocation pointed to by ptr. */ JEMALLOC_ALWAYS_INLINE size_t arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr) { assert(ptr != NULL); - return index2size(extent_szind_get(extent)); + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)ptr, true); + assert(szind != NSIZES); + + if (config_debug && unlikely(extent != NULL)) { + rtree_leaf_elm_t elm; + rtree_leaf_elm_read(rtree_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)ptr, true), true, &elm); + + assert(extent == rtree_leaf_elm_extent_get(&elm)); + assert(szind == extent_szind_get(extent)); + } + + return index2size(szind); +} + +JEMALLOC_ALWAYS_INLINE size_t +arena_vsalloc(tsdn_t *tsdn, const void *ptr) { + /* + * Return 0 if ptr is not within an extent managed by jemalloc. This + * function has two extra costs relative to isalloc(): + * - The rtree calls cannot claim to be dependent lookups, which induces + * rtree lookup load dependencies. + * - The lookup may fail, so there is an extra branch to check for + * failure. + */ + + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + extent_t *extent; + szind_t szind; + if (rtree_extent_szind_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)ptr, false, &extent, &szind)) { + return 0; + } + + if (extent == NULL) { + return 0; + } + assert(extent_state_get(extent) == extent_state_active); + /* Only slab members should be looked up via interior pointers. */ + assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); + + assert(szind != NSIZES); + + return index2size(szind); } JEMALLOC_ALWAYS_INLINE void diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent_externs.h index 68c49a17..6c153d04 100644 --- a/include/jemalloc/internal/extent_externs.h +++ b/include/jemalloc/internal/extent_externs.h @@ -54,7 +54,7 @@ bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, size_t length); extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, - szind_t szind_a, size_t size_b, szind_t szind_b); + szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b); bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b); diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h index 549c8f2f..56f306df 100644 --- a/include/jemalloc/internal/extent_inlines.h +++ b/include/jemalloc/internal/extent_inlines.h @@ -2,11 +2,11 @@ #define JEMALLOC_INTERNAL_EXTENT_INLINES_H #ifndef JEMALLOC_ENABLE_INLINE -extent_t *extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent); arena_t *extent_arena_get(const extent_t *extent); void *extent_base_get(const extent_t *extent); void *extent_addr_get(const extent_t *extent); size_t extent_size_get(const extent_t *extent); +szind_t extent_szind_get_maybe_invalid(const extent_t *extent); szind_t extent_szind_get(const extent_t *extent); size_t extent_usize_get(const extent_t *extent); void *extent_before_get(const extent_t *extent); @@ -47,15 +47,6 @@ int extent_snad_comp(const extent_t *a, const extent_t *b); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_)) -JEMALLOC_INLINE extent_t * -extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent) { - rtree_ctx_t rtree_ctx_fallback; - rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - - return rtree_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, - dependent); -} - JEMALLOC_INLINE arena_t * extent_arena_get(const extent_t *extent) { return extent->e_arena; @@ -81,11 +72,18 @@ extent_size_get(const extent_t *extent) { } JEMALLOC_INLINE szind_t -extent_szind_get(const extent_t *extent) { - assert(extent->e_szind < NSIZES); /* Never call when "invalid". */ +extent_szind_get_maybe_invalid(const extent_t *extent) { + assert(extent->e_szind <= NSIZES); return extent->e_szind; } +JEMALLOC_INLINE szind_t +extent_szind_get(const extent_t *extent) { + szind_t szind = extent_szind_get_maybe_invalid(extent); + assert(szind < NSIZES); /* Never call when "invalid". */ + return szind; +} + JEMALLOC_INLINE size_t extent_usize_get(const extent_t *extent) { return index2size(extent_szind_get(extent)); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index b184380d..238ebdca 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -535,7 +535,6 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/tsd_inlines.h" #include "jemalloc/internal/witness_inlines.h" #include "jemalloc/internal/mutex_inlines.h" -#include "jemalloc/internal/rtree_inlines.h" #ifndef JEMALLOC_ENABLE_INLINE pszind_t psz2ind(size_t psz); @@ -934,6 +933,7 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) { #endif #include "jemalloc/internal/extent_inlines.h" +#include "jemalloc/internal/rtree_inlines.h" #include "jemalloc/internal/base_inlines.h" #include "jemalloc/internal/bitmap_inlines.h" /* @@ -994,7 +994,11 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) { JEMALLOC_ALWAYS_INLINE extent_t * iealloc(tsdn_t *tsdn, const void *ptr) { - return extent_lookup(tsdn, ptr, true); + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + return rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)ptr, true); } #endif @@ -1113,25 +1117,7 @@ ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { JEMALLOC_ALWAYS_INLINE size_t ivsalloc(tsdn_t *tsdn, const void *ptr) { - extent_t *extent; - - /* - * Return 0 if ptr is not within an extent managed by jemalloc. This - * function has two extra costs relative to isalloc(): - * - The extent_lookup() call cannot claim to be a dependent lookup, - * which induces rtree lookup load dependencies. - * - The lookup may fail, so there is an extra branch to check for - * failure. - * */ - extent = extent_lookup(tsdn, ptr, false); - if (extent == NULL) { - return 0; - } - assert(extent_state_get(extent) == extent_state_active); - /* Only slab members should be looked up via interior pointers. */ - assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); - - return isalloc(tsdn, extent, ptr); + return arena_vsalloc(tsdn, ptr); } JEMALLOC_ALWAYS_INLINE void diff --git a/include/jemalloc/internal/large_externs.h b/include/jemalloc/internal/large_externs.h index 66aa755c..2a208c83 100644 --- a/include/jemalloc/internal/large_externs.h +++ b/include/jemalloc/internal/large_externs.h @@ -14,8 +14,8 @@ extern large_dalloc_junk_t *large_dalloc_junk; typedef void (large_dalloc_maybe_junk_t)(void *, size_t); extern large_dalloc_maybe_junk_t *large_dalloc_maybe_junk; #else -void large_dalloc_junk(void *ptr, size_t usize); -void large_dalloc_maybe_junk(void *ptr, size_t usize); +void large_dalloc_junk(void *ptr, size_t size); +void large_dalloc_maybe_junk(void *ptr, size_t size); #endif void large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent); void large_dalloc_finish(tsdn_t *tsdn, extent_t *extent); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index a0deef89..169e7d11 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -77,6 +77,7 @@ arena_stats_merge arena_tcache_fill_small arena_tdata_get arena_tdata_get_hard +arena_vsalloc arenas arenas_tdata_cleanup b0get @@ -169,7 +170,6 @@ extent_list_init extent_list_last extent_list_remove extent_list_replace -extent_lookup extent_merge_wrapper extent_past_get extent_prof_tctx_get @@ -192,6 +192,7 @@ extent_split_wrapper extent_state_get extent_state_set extent_szind_get +extent_szind_get_maybe_invalid extent_szind_set extent_usize_get extent_zeroed_get @@ -413,25 +414,33 @@ psz2ind psz2u rtree_clear rtree_delete +rtree_extent_read +rtree_extent_szind_read rtree_leaf_alloc rtree_leaf_dalloc rtree_leaf_elm_acquire +rtree_leaf_elm_extent_read +rtree_leaf_elm_extent_write rtree_leaf_elm_lookup rtree_leaf_elm_lookup_hard -rtree_leaf_elm_read -rtree_leaf_elm_read_acquired rtree_leaf_elm_release +rtree_leaf_elm_slab_read +rtree_leaf_elm_slab_write +rtree_leaf_elm_szind_read +rtree_leaf_elm_szind_write rtree_leaf_elm_witness_access rtree_leaf_elm_witness_acquire rtree_leaf_elm_witness_release rtree_leaf_elm_write -rtree_leaf_elm_write_acquired rtree_leafkey rtree_new rtree_node_alloc rtree_node_dalloc rtree_read rtree_subkey +rtree_szind_read +rtree_szind_slab_read +rtree_szind_slab_update rtree_write s2u s2u_compute diff --git a/include/jemalloc/internal/rtree_inlines.h b/include/jemalloc/internal/rtree_inlines.h index 3af17d36..9c337b85 100644 --- a/include/jemalloc/internal/rtree_inlines.h +++ b/include/jemalloc/internal/rtree_inlines.h @@ -4,21 +4,40 @@ #ifndef JEMALLOC_ENABLE_INLINE uintptr_t rtree_leafkey(uintptr_t key); uintptr_t rtree_subkey(uintptr_t key, unsigned level); -extent_t *rtree_leaf_elm_read(rtree_leaf_elm_t *elm, bool dependent); -void rtree_leaf_elm_write(rtree_leaf_elm_t *elm, const extent_t *extent); +extent_t *rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool acquired, bool dependent); +szind_t rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool acquired, bool dependent); +bool rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool acquired, bool dependent); +void rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool acquired, extent_t *extent); +void rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool acquired, szind_t szind); +void rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool acquired, bool slab); +void rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool acquired, extent_t *extent, szind_t szind, bool slab); rtree_leaf_elm_t *rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing); bool rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key, const extent_t *extent); -extent_t *rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key, bool dependent); + uintptr_t key, extent_t *extent, szind_t szind, bool slab); +rtree_leaf_elm_t *rtree_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent); +extent_t *rtree_extent_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent); +szind_t rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent); +bool rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, extent_t **r_extent, + szind_t *r_szind); +bool rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab); +void rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, + rtree_ctx_t *rtree_ctx, uintptr_t key, szind_t szind, bool slab); rtree_leaf_elm_t *rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing); -extent_t *rtree_leaf_elm_read_acquired(tsdn_t *tsdn, const rtree_t *rtree, - rtree_leaf_elm_t *elm); -void rtree_leaf_elm_write_acquired(tsdn_t *tsdn, const rtree_t *rtree, - rtree_leaf_elm_t *elm, const extent_t *extent); -void rtree_leaf_elm_release(tsdn_t *tsdn, const rtree_t *rtree, +void rtree_leaf_elm_release(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm); void rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key); @@ -45,38 +64,104 @@ rtree_subkey(uintptr_t key, unsigned level) { return ((key >> shiftbits) & mask); } +/* + * Atomic getters. + * + * dependent: Reading a value on behalf of a pointer to a valid allocation + * is guaranteed to be a clean read even without synchronization, + * because the rtree update became visible in memory before the + * pointer came into existence. + * !dependent: An arbitrary read, e.g. on behalf of ivsalloc(), may not be + * dependent on a previous rtree write, which means a stale read + * could result if synchronization were omitted here. + */ JEMALLOC_ALWAYS_INLINE extent_t * -rtree_leaf_elm_read(rtree_leaf_elm_t *elm, bool dependent) { - extent_t *extent; - - if (dependent) { - /* - * Reading a value on behalf of a pointer to a valid allocation - * is guaranteed to be a clean read even without - * synchronization, because the rtree update became visible in - * memory before the pointer came into existence. - */ - extent = (extent_t *)atomic_load_p(&elm->extent, - ATOMIC_RELAXED); - } else { - /* - * An arbitrary read, e.g. on behalf of ivsalloc(), may not be - * dependent on a previous rtree write, which means a stale read - * could result if synchronization were omitted here. - */ - extent = (extent_t *)atomic_load_p(&elm->extent, - ATOMIC_ACQUIRE); +rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool acquired, bool dependent) { + if (config_debug && acquired) { + assert(dependent); + rtree_leaf_elm_witness_access(tsdn, rtree, elm); } - /* Mask the lock bit. */ + extent_t *extent = (extent_t *)atomic_load_p(&elm->le_extent, dependent + ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); + assert(!acquired || ((uintptr_t)extent & (uintptr_t)0x1) == + (uintptr_t)0x1); + /* Mask lock bit. */ extent = (extent_t *)((uintptr_t)extent & ~((uintptr_t)0x1)); - return extent; } +JEMALLOC_ALWAYS_INLINE szind_t +rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool acquired, bool dependent) { + if (config_debug && acquired) { + assert(dependent); + rtree_leaf_elm_witness_access(tsdn, rtree, elm); + } + + return (szind_t)atomic_load_u(&elm->le_szind, dependent ? ATOMIC_RELAXED + : ATOMIC_ACQUIRE); +} + +JEMALLOC_ALWAYS_INLINE bool +rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool acquired, bool dependent) { + if (config_debug && acquired) { + assert(dependent); + rtree_leaf_elm_witness_access(tsdn, rtree, elm); + } + + return atomic_load_b(&elm->le_slab, dependent ? ATOMIC_RELAXED : + ATOMIC_ACQUIRE); +} + JEMALLOC_INLINE void -rtree_leaf_elm_write(rtree_leaf_elm_t *elm, const extent_t *extent) { - atomic_store_p(&elm->extent, (void *)extent, ATOMIC_RELEASE); +rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool acquired, extent_t *extent) { + if (config_debug && acquired) { + rtree_leaf_elm_witness_access(tsdn, rtree, elm); + } + assert(((uintptr_t)extent & (uintptr_t)0x1) == (uintptr_t)0x0); + + if (acquired) { + /* Overlay lock bit. */ + extent = (extent_t *)((uintptr_t)extent | (uintptr_t)0x1); + } + atomic_store_p(&elm->le_extent, extent, ATOMIC_RELEASE); +} + +JEMALLOC_INLINE void +rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool acquired, szind_t szind) { + if (config_debug && acquired) { + rtree_leaf_elm_witness_access(tsdn, rtree, elm); + } + assert(szind <= NSIZES); + + atomic_store_u(&elm->le_szind, szind, ATOMIC_RELEASE); +} + +JEMALLOC_INLINE void +rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool acquired, bool slab) { + if (config_debug && acquired) { + rtree_leaf_elm_witness_access(tsdn, rtree, elm); + } + + atomic_store_b(&elm->le_slab, slab, ATOMIC_RELEASE); +} + +JEMALLOC_INLINE void +rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool acquired, extent_t *extent, szind_t szind, bool slab) { + rtree_leaf_elm_slab_write(tsdn, rtree, elm, acquired, slab); + rtree_leaf_elm_szind_write(tsdn, rtree, elm, acquired, szind); + /* + * Write extent last, since the element is atomically considered valid + * as soon as the extent field is non-NULL. + */ + rtree_leaf_elm_extent_write(tsdn, rtree, elm, acquired, extent); } JEMALLOC_ALWAYS_INLINE rtree_leaf_elm_t * @@ -124,34 +209,99 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, JEMALLOC_INLINE bool rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, - const extent_t *extent) { - rtree_leaf_elm_t *elm; + extent_t *extent, szind_t szind, bool slab) { + /* Use rtree_clear() to set the extent to NULL. */ + assert(extent != NULL); - assert(extent != NULL); /* Use rtree_clear() for this case. */ - assert(((uintptr_t)extent & (uintptr_t)0x1) == (uintptr_t)0x0); - - elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, key, false, true); + rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, + key, false, true); if (elm == NULL) { return true; } - assert(rtree_leaf_elm_read(elm, false) == NULL); - rtree_leaf_elm_write(elm, extent); + + assert(rtree_leaf_elm_extent_read(tsdn, rtree, elm, false, false) == + NULL); + rtree_leaf_elm_write(tsdn, rtree, elm, false, extent, szind, slab); return false; } -JEMALLOC_ALWAYS_INLINE extent_t * +JEMALLOC_ALWAYS_INLINE rtree_leaf_elm_t * rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent) { - rtree_leaf_elm_t *elm; - - elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, key, dependent, - false); + rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, + key, dependent, false); if (!dependent && elm == NULL) { return NULL; } + assert(elm != NULL); + return elm; +} - return rtree_leaf_elm_read(elm, dependent); +JEMALLOC_ALWAYS_INLINE extent_t * +rtree_extent_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, bool dependent) { + rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, + dependent); + if (!dependent && elm == NULL) { + return NULL; + } + return rtree_leaf_elm_extent_read(tsdn, rtree, elm, false, dependent); +} + +JEMALLOC_ALWAYS_INLINE szind_t +rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, bool dependent) { + rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, + dependent); + if (!dependent && elm == NULL) { + return NSIZES; + } + return rtree_leaf_elm_szind_read(tsdn, rtree, elm, false, dependent); +} + +/* + * rtree_slab_read() is intentionally omitted because slab is always read in + * conjunction with szind, which makes rtree_szind_slab_read() a better choice. + */ + +JEMALLOC_ALWAYS_INLINE bool +rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, bool dependent, extent_t **r_extent, szind_t *r_szind) { + rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, + dependent); + if (!dependent && elm == NULL) { + return true; + } + *r_extent = rtree_leaf_elm_extent_read(tsdn, rtree, elm, false, + dependent); + *r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, false, + dependent); + return false; +} + +JEMALLOC_ALWAYS_INLINE bool +rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) { + rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, + dependent); + if (!dependent && elm == NULL) { + return true; + } + *r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, false, + dependent); + *r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, false, dependent); + return false; +} + +JEMALLOC_INLINE void +rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, szind_t szind, bool slab) { + assert(!slab || szind < NBINS); + + rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); + rtree_leaf_elm_slab_write(tsdn, rtree, elm, false, slab); + rtree_leaf_elm_szind_write(tsdn, rtree, elm, false, szind); } JEMALLOC_INLINE rtree_leaf_elm_t * @@ -162,18 +312,19 @@ rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, if (!dependent && elm == NULL) { return NULL; } + assert(elm != NULL); spin_t spinner = SPIN_INITIALIZER; while (true) { /* The least significant bit serves as a lock. */ - void *extent_and_lock = atomic_load_p(&elm->extent, + void *extent_and_lock = atomic_load_p(&elm->le_extent, ATOMIC_RELAXED); if (likely(((uintptr_t)extent_and_lock & (uintptr_t)0x1) == 0)) { void *locked = (void *)((uintptr_t)extent_and_lock | (uintptr_t)0x1); if (likely(atomic_compare_exchange_strong_p( - &elm->extent, &extent_and_lock, locked, + &elm->le_extent, &extent_and_lock, locked, ATOMIC_ACQUIRE, ATOMIC_RELAXED))) { break; } @@ -188,42 +339,11 @@ rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, return elm; } -JEMALLOC_INLINE extent_t * -rtree_leaf_elm_read_acquired(tsdn_t *tsdn, const rtree_t *rtree, - rtree_leaf_elm_t *elm) { - extent_t *extent; - void *ptr = atomic_load_p(&elm->extent, ATOMIC_RELAXED); - assert(((uintptr_t)ptr & (uintptr_t)0x1) == (uintptr_t)0x1); - extent = (extent_t *)((uintptr_t)ptr & ~((uintptr_t)0x1)); - assert(((uintptr_t)extent & (uintptr_t)0x1) == (uintptr_t)0x0); - - if (config_debug) { - rtree_leaf_elm_witness_access(tsdn, rtree, elm); - } - - return extent; -} - JEMALLOC_INLINE void -rtree_leaf_elm_write_acquired(tsdn_t *tsdn, const rtree_t *rtree, - rtree_leaf_elm_t *elm, const extent_t *extent) { - assert(((uintptr_t)extent & (uintptr_t)0x1) == (uintptr_t)0x0); - assert(((uintptr_t)atomic_load_p(&elm->extent, ATOMIC_RELAXED) - & (uintptr_t)0x1) == (uintptr_t)0x1); - - if (config_debug) { - rtree_leaf_elm_witness_access(tsdn, rtree, elm); - } - atomic_store_p(&elm->extent, (void *)((uintptr_t)extent | - (uintptr_t)0x1), ATOMIC_RELEASE); - assert(rtree_leaf_elm_read_acquired(tsdn, rtree, elm) == extent); -} - -JEMALLOC_INLINE void -rtree_leaf_elm_release(tsdn_t *tsdn, const rtree_t *rtree, - rtree_leaf_elm_t *elm) { - rtree_leaf_elm_write(elm, rtree_leaf_elm_read_acquired(tsdn, rtree, - elm)); +rtree_leaf_elm_release(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm) { + extent_t *extent = rtree_leaf_elm_extent_read(tsdn, rtree, elm, true, + true); + rtree_leaf_elm_extent_write(tsdn, rtree, elm, false, extent); if (config_debug) { rtree_leaf_elm_witness_release(tsdn, rtree, elm); } @@ -232,10 +352,9 @@ rtree_leaf_elm_release(tsdn_t *tsdn, const rtree_t *rtree, JEMALLOC_INLINE void rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) { - rtree_leaf_elm_t *elm; - - elm = rtree_leaf_elm_acquire(tsdn, rtree, rtree_ctx, key, true, false); - rtree_leaf_elm_write_acquired(tsdn, rtree, elm, NULL); + rtree_leaf_elm_t *elm = rtree_leaf_elm_acquire(tsdn, rtree, rtree_ctx, + key, true, false); + rtree_leaf_elm_write(tsdn, rtree, elm, true, NULL, NSIZES, false); rtree_leaf_elm_release(tsdn, rtree, elm); } #endif diff --git a/include/jemalloc/internal/rtree_structs.h b/include/jemalloc/internal/rtree_structs.h index 68554035..e9a507ab 100644 --- a/include/jemalloc/internal/rtree_structs.h +++ b/include/jemalloc/internal/rtree_structs.h @@ -2,11 +2,13 @@ #define JEMALLOC_INTERNAL_RTREE_STRUCTS_H struct rtree_node_elm_s { - atomic_p_t child; + atomic_p_t child; /* (rtree_{node,leaf}_elm_t *) */ }; struct rtree_leaf_elm_s { - atomic_p_t extent; + atomic_p_t le_extent; /* (extent_t *) */ + atomic_u_t le_szind; /* (szind_t) */ + atomic_b_t le_slab; /* (bool) */ }; struct rtree_leaf_elm_witness_s { diff --git a/src/arena.c b/src/arena.c index 968343c0..2dd84761 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1468,7 +1468,12 @@ arena_prof_promote(tsdn_t *tsdn, extent_t *extent, const void *ptr, assert(isalloc(tsdn, extent, ptr) == LARGE_MINCLASS); assert(usize <= SMALL_MAXCLASS); - extent_szind_set(extent, size2index(usize)); + szind_t szind = size2index(usize); + extent_szind_set(extent, szind); + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, + szind, false); prof_accum_cancel(tsdn, &arena->prof_accum, usize); @@ -1481,6 +1486,10 @@ arena_prof_demote(tsdn_t *tsdn, extent_t *extent, const void *ptr) { assert(ptr != NULL); extent_szind_set(extent, NBINS); + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, + NBINS, false); assert(isalloc(tsdn, extent, ptr) == LARGE_MINCLASS); diff --git a/src/extent.c b/src/extent.c index 2a16d387..a95e2748 100644 --- a/src/extent.c +++ b/src/extent.c @@ -476,11 +476,12 @@ extent_rtree_acquire(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, static void extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a, - rtree_leaf_elm_t *elm_b, const extent_t *extent) { - rtree_leaf_elm_write_acquired(tsdn, &extents_rtree, elm_a, extent); + rtree_leaf_elm_t *elm_b, extent_t *extent, szind_t szind, bool slab) { + rtree_leaf_elm_write(tsdn, &extents_rtree, elm_a, true, extent, szind, + slab); if (elm_b != NULL) { - rtree_leaf_elm_write_acquired(tsdn, &extents_rtree, elm_b, - extent); + rtree_leaf_elm_write(tsdn, &extents_rtree, elm_b, true, extent, + szind, slab); } } @@ -494,16 +495,15 @@ extent_rtree_release(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a, } static void -extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, - const extent_t *extent) { - size_t i; - +extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, extent_t *extent, + szind_t szind) { assert(extent_slab_get(extent)); - for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) { + /* Register interior. */ + for (size_t i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) { rtree_write(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_base_get(extent) + (uintptr_t)(i << - LG_PAGE), extent); + LG_PAGE), extent, szind, true); } } @@ -542,7 +542,7 @@ extent_gdump_sub(tsdn_t *tsdn, const extent_t *extent) { } static bool -extent_register_impl(tsdn_t *tsdn, const extent_t *extent, bool gdump_add) { +extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool gdump_add) { rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_leaf_elm_t *elm_a, *elm_b; @@ -551,9 +551,11 @@ extent_register_impl(tsdn_t *tsdn, const extent_t *extent, bool gdump_add) { &elm_b)) { return true; } - extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent); - if (extent_slab_get(extent)) { - extent_interior_register(tsdn, rtree_ctx, extent); + szind_t szind = extent_szind_get_maybe_invalid(extent); + bool slab = extent_slab_get(extent); + extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent, szind, slab); + if (slab) { + extent_interior_register(tsdn, rtree_ctx, extent, szind); } extent_rtree_release(tsdn, elm_a, elm_b); @@ -565,24 +567,24 @@ extent_register_impl(tsdn_t *tsdn, const extent_t *extent, bool gdump_add) { } static bool -extent_register(tsdn_t *tsdn, const extent_t *extent) { +extent_register(tsdn_t *tsdn, extent_t *extent) { return extent_register_impl(tsdn, extent, true); } static bool -extent_register_no_gdump_add(tsdn_t *tsdn, const extent_t *extent) { +extent_register_no_gdump_add(tsdn_t *tsdn, extent_t *extent) { return extent_register_impl(tsdn, extent, false); } static void -extent_reregister(tsdn_t *tsdn, const extent_t *extent) { +extent_reregister(tsdn_t *tsdn, extent_t *extent) { bool err = extent_register(tsdn, extent); assert(!err); } static void extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, - const extent_t *extent) { + extent_t *extent) { size_t i; assert(extent_slab_get(extent)); @@ -602,7 +604,7 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent) { extent_rtree_acquire(tsdn, rtree_ctx, extent, true, false, &elm_a, &elm_b); - extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL); + extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, NSIZES, false); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -653,13 +655,12 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena, r_extent_hooks); extent_t *extent; if (new_addr != NULL) { - rtree_leaf_elm_t *elm; - - elm = rtree_leaf_elm_acquire(tsdn, &extents_rtree, rtree_ctx, - (uintptr_t)new_addr, false, false); + rtree_leaf_elm_t *elm = rtree_leaf_elm_acquire(tsdn, + &extents_rtree, rtree_ctx, (uintptr_t)new_addr, false, + false); if (elm != NULL) { - extent = rtree_leaf_elm_read_acquired(tsdn, - &extents_rtree, elm); + extent = rtree_leaf_elm_extent_read(tsdn, + &extents_rtree, elm, true, true); if (extent != NULL) { assert(extent_base_get(extent) == new_addr); if (extent_arena_get(extent) != arena || @@ -715,7 +716,8 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, if (leadsize != 0) { extent_t *lead = extent; extent = extent_split_wrapper(tsdn, arena, r_extent_hooks, - lead, leadsize, NSIZES, esize + trailsize, szind); + lead, leadsize, NSIZES, false, esize + trailsize, szind, + slab); if (extent == NULL) { extent_deregister(tsdn, lead); extents_leak(tsdn, arena, r_extent_hooks, extents, @@ -728,7 +730,8 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, /* Split the trail. */ if (trailsize != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, - r_extent_hooks, extent, esize, szind, trailsize, NSIZES); + r_extent_hooks, extent, esize, szind, slab, trailsize, + NSIZES, false); if (trail == NULL) { extent_deregister(tsdn, extent); extents_leak(tsdn, arena, r_extent_hooks, extents, @@ -742,6 +745,16 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, * splitting occurred. */ extent_szind_set(extent, szind); + if (szind != NSIZES) { + rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)extent_addr_get(extent), szind, slab); + if (slab && extent_size_get(extent) > PAGE) { + rtree_szind_slab_update(tsdn, &extents_rtree, + rtree_ctx, + (uintptr_t)extent_past_get(extent) - + (uintptr_t)PAGE, szind, slab); + } + } } return extent; @@ -788,7 +801,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, assert(extent_state_get(extent) == extent_state_active); if (slab) { extent_slab_set(extent, slab); - extent_interior_register(tsdn, rtree_ctx, extent); + extent_interior_register(tsdn, rtree_ctx, extent, szind); } if (*zero) { @@ -934,7 +947,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, if (leadsize != 0) { extent_t *lead = extent; extent = extent_split_wrapper(tsdn, arena, r_extent_hooks, lead, - leadsize, NSIZES, esize + trailsize, szind); + leadsize, NSIZES, false, esize + trailsize, szind, slab); if (extent == NULL) { extent_deregister(tsdn, lead); extents_leak(tsdn, arena, r_extent_hooks, false, lead); @@ -947,7 +960,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, /* Split the trail. */ if (trailsize != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, - r_extent_hooks, extent, esize, szind, trailsize, NSIZES); + r_extent_hooks, extent, esize, szind, slab, trailsize, + NSIZES, false); if (trail == NULL) { extent_deregister(tsdn, extent); extents_leak(tsdn, arena, r_extent_hooks, @@ -961,7 +975,21 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, * Splitting causes szind to be set as a side effect, but no * splitting occurred. */ + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, + &rtree_ctx_fallback); + extent_szind_set(extent, szind); + if (szind != NSIZES) { + rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)extent_addr_get(extent), szind, slab); + if (slab && extent_size_get(extent) > PAGE) { + rtree_szind_slab_update(tsdn, &extents_rtree, + rtree_ctx, + (uintptr_t)extent_past_get(extent) - + (uintptr_t)PAGE, szind, slab); + } + } } if (*commit && !extent_committed_get(extent)) { @@ -987,7 +1015,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, &rtree_ctx_fallback); extent_slab_set(extent, true); - extent_interior_register(tsdn, rtree_ctx, extent); + extent_interior_register(tsdn, rtree_ctx, extent, szind); } if (*zero && !extent_zeroed_get(extent)) { void *addr = extent_base_get(extent); @@ -1162,8 +1190,8 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, &extents_rtree, rtree_ctx, (uintptr_t)extent_past_get(extent), false, false); if (next_elm != NULL) { - extent_t *next = rtree_leaf_elm_read_acquired(tsdn, - &extents_rtree, next_elm); + extent_t *next = rtree_leaf_elm_extent_read(tsdn, + &extents_rtree, next_elm, true, true); /* * extents->mtx only protects against races for * like-state extents, so call extent_can_coalesce() @@ -1188,8 +1216,8 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, &extents_rtree, rtree_ctx, (uintptr_t)extent_before_get(extent), false, false); if (prev_elm != NULL) { - extent_t *prev = rtree_leaf_elm_read_acquired(tsdn, - &extents_rtree, prev_elm); + extent_t *prev = rtree_leaf_elm_extent_read(tsdn, + &extents_rtree, prev_elm, true, true); bool can_coalesce = (prev != NULL && extent_can_coalesce(arena, extents, extent, prev)); rtree_leaf_elm_release(tsdn, &extents_rtree, prev_elm); @@ -1231,7 +1259,8 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_slab_set(extent, false); } - assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent); + assert(rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)extent_base_get(extent), true) == extent); if (!extents->delay_coalesce) { extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, @@ -1467,7 +1496,7 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, extent_t * extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, - szind_t szind_a, size_t size_b, szind_t szind_b) { + szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) { assert(extent_size_get(extent) == size_a + size_b); witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); @@ -1491,7 +1520,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t lead; extent_init(&lead, arena, extent_addr_get(extent), size_a, - extent_slab_get(extent), szind_a, extent_sn_get(extent), + slab_a, szind_a, extent_sn_get(extent), extent_state_get(extent), extent_zeroed_get(extent), extent_committed_get(extent)); @@ -1502,9 +1531,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, } extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + - size_a), size_b, extent_slab_get(extent), szind_b, - extent_sn_get(extent), extent_state_get(extent), - extent_zeroed_get(extent), extent_committed_get(extent)); + size_a), size_b, slab_b, szind_b, extent_sn_get(extent), + extent_state_get(extent), extent_zeroed_get(extent), + extent_committed_get(extent)); if (extent_rtree_acquire(tsdn, rtree_ctx, trail, false, true, &trail_elm_a, &trail_elm_b)) { goto label_error_c; @@ -1519,8 +1548,10 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_size_set(extent, size_a); extent_szind_set(extent, szind_a); - extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent); - extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail); + extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent, + szind_a, slab_a); + extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail, + szind_b, slab_b); extent_rtree_release(tsdn, lead_elm_a, lead_elm_b); extent_rtree_release(tsdn, trail_elm_a, trail_elm_b); @@ -1599,13 +1630,13 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, &b_elm_b); if (a_elm_b != NULL) { - rtree_leaf_elm_write_acquired(tsdn, &extents_rtree, a_elm_b, - NULL); + rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, true, NULL, + NSIZES, false); rtree_leaf_elm_release(tsdn, &extents_rtree, a_elm_b); } if (b_elm_b != NULL) { - rtree_leaf_elm_write_acquired(tsdn, &extents_rtree, b_elm_a, - NULL); + rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, true, NULL, + NSIZES, false); rtree_leaf_elm_release(tsdn, &extents_rtree, b_elm_a); } else { b_elm_b = b_elm_a; @@ -1617,7 +1648,7 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_sn_get(a) : extent_sn_get(b)); extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b)); - extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a); + extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, NSIZES, false); extent_rtree_release(tsdn, a_elm_a, b_elm_b); extent_dalloc(tsdn, extent_arena_get(b), b); diff --git a/src/large.c b/src/large.c index 0e9f0d72..845202f9 100644 --- a/src/large.c +++ b/src/large.c @@ -66,8 +66,8 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, #define large_dalloc_junk JEMALLOC_N(n_large_dalloc_junk) #endif void -large_dalloc_junk(void *ptr, size_t usize) { - memset(ptr, JEMALLOC_FREE_JUNK, usize); +large_dalloc_junk(void *ptr, size_t size) { + memset(ptr, JEMALLOC_FREE_JUNK, size); } #ifdef JEMALLOC_JET #undef large_dalloc_junk @@ -80,14 +80,14 @@ large_dalloc_junk_t *large_dalloc_junk = JEMALLOC_N(n_large_dalloc_junk); #define large_dalloc_maybe_junk JEMALLOC_N(n_large_dalloc_maybe_junk) #endif void -large_dalloc_maybe_junk(void *ptr, size_t usize) { +large_dalloc_maybe_junk(void *ptr, size_t size) { if (config_fill && have_dss && unlikely(opt_junk_free)) { /* * Only bother junk filling if the extent isn't about to be * unmapped. */ if (!config_munmap || (have_dss && extent_in_dss(ptr))) { - large_dalloc_junk(ptr, usize); + large_dalloc_junk(ptr, size); } } } @@ -115,7 +115,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { if (diff != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, &extent_hooks, extent, usize + large_pad, size2index(usize), - diff, NSIZES); + false, diff, NSIZES, false); if (trail == NULL) { return true; } @@ -182,7 +182,12 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, extent_dalloc_wrapper(tsdn, arena, &extent_hooks, trail); return true; } - extent_szind_set(extent, size2index(usize)); + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + szind_t szind = size2index(usize); + extent_szind_set(extent, szind); + rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)extent_addr_get(extent), szind, false); if (config_stats && new_mapping) { arena_stats_mapped_add(tsdn, &arena->stats, trailsize); diff --git a/test/unit/arena_reset.c b/test/unit/arena_reset.c index 24c7f526..589c652c 100644 --- a/test/unit/arena_reset.c +++ b/test/unit/arena_reset.c @@ -57,9 +57,16 @@ get_large_size(size_t ind) { /* Like ivsalloc(), but safe to call on discarded allocations. */ static size_t vsalloc(tsdn_t *tsdn, const void *ptr) { - extent_t *extent; + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + extent_t *extent; + szind_t szind; + if (rtree_extent_szind_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)ptr, false, &extent, &szind)) { + return 0; + } - extent = extent_lookup(tsdn, ptr, false); if (extent == NULL) { return 0; } @@ -67,7 +74,11 @@ vsalloc(tsdn_t *tsdn, const void *ptr) { return 0; } - return isalloc(tsdn, extent, ptr); + if (szind == NSIZES) { + return 0; + } + + return index2size(szind); } static unsigned diff --git a/test/unit/rtree.c b/test/unit/rtree.c index 221f2f2c..b04e321b 100644 --- a/test/unit/rtree.c +++ b/test/unit/rtree.c @@ -70,8 +70,8 @@ TEST_BEGIN(test_rtree_read_empty) { rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; test_rtree = &rtree; assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure"); - assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx, PAGE, false), - "rtree_read() should return NULL for empty tree"); + assert_ptr_null(rtree_extent_read(tsdn, &rtree, &rtree_ctx, PAGE, + false), "rtree_extent_read() should return NULL for empty tree"); rtree_delete(tsdn, &rtree); test_rtree = NULL; } @@ -99,6 +99,8 @@ thd_start(void *varg) { sfmt = init_gen_rand(arg->seed); extent = (extent_t *)malloc(sizeof(extent)); assert_ptr_not_null(extent, "Unexpected malloc() failure"); + extent_init(extent, NULL, NULL, 0, false, NSIZES, 0, + extent_state_active, false, false); tsdn = tsdn_fetch(); for (i = 0; i < NITERS; i++) { @@ -109,18 +111,24 @@ thd_start(void *varg) { &arg->rtree, &rtree_ctx, key, false, true); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_acquire() failure"); - rtree_leaf_elm_write_acquired(tsdn, &arg->rtree, elm, - extent); + rtree_leaf_elm_write(tsdn, &arg->rtree, elm, true, + extent, NSIZES, false); rtree_leaf_elm_release(tsdn, &arg->rtree, elm); elm = rtree_leaf_elm_acquire(tsdn, &arg->rtree, &rtree_ctx, key, true, false); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_acquire() failure"); - rtree_leaf_elm_read_acquired(tsdn, &arg->rtree, elm); + rtree_leaf_elm_extent_read(tsdn, &arg->rtree, elm, true, + true); + rtree_leaf_elm_szind_read(tsdn, &arg->rtree, elm, true, + true); + rtree_leaf_elm_slab_read(tsdn, &arg->rtree, elm, true, + true); rtree_leaf_elm_release(tsdn, &arg->rtree, elm); } else { - rtree_read(tsdn, &arg->rtree, &rtree_ctx, key, false); + rtree_extent_read(tsdn, &arg->rtree, &rtree_ctx, key, + false); } } @@ -158,26 +166,33 @@ TEST_END TEST_BEGIN(test_rtree_extrema) { extent_t extent_a, extent_b; - tsdn_t *tsdn; + extent_init(&extent_a, NULL, NULL, LARGE_MINCLASS, false, + size2index(LARGE_MINCLASS), 0, extent_state_active, false, false); + extent_init(&extent_b, NULL, NULL, 0, false, NSIZES, 0, + extent_state_active, false, false); - tsdn = tsdn_fetch(); + tsdn_t *tsdn = tsdn_fetch(); rtree_t rtree; rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; test_rtree = &rtree; assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure"); - assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, PAGE, &extent_a), + assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, PAGE, &extent_a, + extent_szind_get(&extent_a), extent_slab_get(&extent_a)), "Unexpected rtree_write() failure"); - assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, PAGE, true), + rtree_szind_slab_update(tsdn, &rtree, &rtree_ctx, PAGE, + extent_szind_get(&extent_a), extent_slab_get(&extent_a)); + assert_ptr_eq(rtree_extent_read(tsdn, &rtree, &rtree_ctx, PAGE, true), &extent_a, - "rtree_read() should return previously set value"); + "rtree_extent_read() should return previously set value"); assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, ~((uintptr_t)0), - &extent_b), "Unexpected rtree_write() failure"); - assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, ~((uintptr_t)0), - true), &extent_b, - "rtree_read() should return previously set value"); + &extent_b, extent_szind_get_maybe_invalid(&extent_b), + extent_slab_get(&extent_b)), "Unexpected rtree_write() failure"); + assert_ptr_eq(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + ~((uintptr_t)0), true), &extent_b, + "rtree_extent_read() should return previously set value"); rtree_delete(tsdn, &rtree); test_rtree = NULL; @@ -191,6 +206,9 @@ TEST_BEGIN(test_rtree_bits) { PAGE + (((uintptr_t)1) << LG_PAGE) - 1}; extent_t extent; + extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, + extent_state_active, false, false); + rtree_t rtree; rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; @@ -200,16 +218,17 @@ TEST_BEGIN(test_rtree_bits) { for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) { assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, keys[i], - &extent), "Unexpected rtree_write() failure"); + &extent, NSIZES, false), + "Unexpected rtree_write() failure"); for (unsigned j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { - assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, - keys[j], true), &extent, - "rtree_read() should return previously set " - "value and ignore insignificant key bits; " - "i=%u, j=%u, set key=%#"FMTxPTR", get " + assert_ptr_eq(rtree_extent_read(tsdn, &rtree, + &rtree_ctx, keys[j], true), + &extent, "rtree_extent_read() should return " + "previously set value and ignore insignificant key " + "bits; i=%u, j=%u, set key=%#"FMTxPTR", get " "key=%#"FMTxPTR, i, j, keys[i], keys[j]); } - assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx, + assert_ptr_null(rtree_extent_read(tsdn, &rtree, &rtree_ctx, (((uintptr_t)2) << LG_PAGE), false), "Only leftmost rtree leaf should be set; i=%u", i); rtree_clear(tsdn, &rtree, &rtree_ctx, keys[i]); @@ -226,10 +245,13 @@ TEST_BEGIN(test_rtree_random) { sfmt_t *sfmt = init_gen_rand(SEED); tsdn_t *tsdn = tsdn_fetch(); uintptr_t keys[NSET]; - extent_t extent; rtree_t rtree; rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; + extent_t extent; + extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, + extent_state_active, false, false); + test_rtree = &rtree; assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure"); @@ -239,26 +261,30 @@ TEST_BEGIN(test_rtree_random) { &rtree_ctx, keys[i], false, true); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_acquire() failure"); - rtree_leaf_elm_write_acquired(tsdn, &rtree, elm, &extent); + rtree_leaf_elm_write(tsdn, &rtree, elm, true, &extent, NSIZES, + false); rtree_leaf_elm_release(tsdn, &rtree, elm); - assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, keys[i], - true), &extent, - "rtree_read() should return previously set value"); + assert_ptr_eq(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + keys[i], true), &extent, + "rtree_extent_read() should return previously set value"); } for (unsigned i = 0; i < NSET; i++) { - assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, keys[i], - true), &extent, - "rtree_read() should return previously set value, i=%u", i); + assert_ptr_eq(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + keys[i], true), &extent, + "rtree_extent_read() should return previously set value, " + "i=%u", i); } for (unsigned i = 0; i < NSET; i++) { rtree_clear(tsdn, &rtree, &rtree_ctx, keys[i]); - assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx, keys[i], - true), "rtree_read() should return previously set value"); + assert_ptr_null(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + keys[i], true), + "rtree_extent_read() should return previously set value"); } for (unsigned i = 0; i < NSET; i++) { - assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx, keys[i], - true), "rtree_read() should return previously set value"); + assert_ptr_null(rtree_extent_read(tsdn, &rtree, &rtree_ctx, + keys[i], true), + "rtree_extent_read() should return previously set value"); } rtree_delete(tsdn, &rtree);