diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 4e20af48..ce9d8b5e 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -212,8 +212,8 @@ struct arena_s { * Heaps of extents that were previously allocated. These are used when * allocating extents, in an attempt to re-use address space. */ - extent_heap_t extents_cached[NPSIZES]; - extent_heap_t extents_retained[NPSIZES]; + extent_heap_t extents_cached[NPSIZES+1]; + extent_heap_t extents_retained[NPSIZES+1]; /* * Ring sentinel used to track unused dirty memory. Dirty memory is * managed as an LRU of cached extents. diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 0e4ffd91..85b34012 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -434,7 +434,7 @@ extern arena_t **arenas; * pind2sz_tab encodes the same information as could be computed by * pind2sz_compute(). */ -extern size_t const pind2sz_tab[NPSIZES]; +extern size_t const pind2sz_tab[NPSIZES+1]; /* * index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by index2size_compute(). @@ -516,9 +516,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/large.h" #ifndef JEMALLOC_ENABLE_INLINE -pszind_t psz2ind_impl(size_t psz, bool clamp); pszind_t psz2ind(size_t psz); -pszind_t psz2ind_clamp(size_t psz); size_t pind2sz_compute(pszind_t pind); size_t pind2sz_lookup(pszind_t pind); size_t pind2sz(pszind_t pind); @@ -544,11 +542,11 @@ ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) JEMALLOC_ALWAYS_INLINE pszind_t -psz2ind_impl(size_t psz, bool clamp) +psz2ind(size_t psz) { if (unlikely(psz > LARGE_MAXCLASS)) - return (clamp ? NPSIZES-1 : NPSIZES); + return (NPSIZES); { pszind_t x = lg_floor((psz<<1)-1); pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - @@ -567,24 +565,12 @@ psz2ind_impl(size_t psz, bool clamp) } } -JEMALLOC_INLINE pszind_t -psz2ind(size_t psz) -{ - - return (psz2ind_impl(psz, false)); -} - -JEMALLOC_INLINE pszind_t -psz2ind_clamp(size_t psz) -{ - - return (psz2ind_impl(psz, true)); -} - JEMALLOC_INLINE size_t pind2sz_compute(pszind_t pind) { + if (unlikely(pind == NPSIZES)) + return (LARGE_MAXCLASS + PAGE); { size_t grp = pind >> LG_SIZE_CLASS_GROUP; size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); @@ -614,7 +600,7 @@ JEMALLOC_INLINE size_t pind2sz(pszind_t pind) { - assert(pind < NPSIZES); + assert(pind < NPSIZES+1); return (pind2sz_lookup(pind)); } @@ -623,7 +609,7 @@ psz2u(size_t psz) { if (unlikely(psz > LARGE_MAXCLASS)) - return (0); + return (LARGE_MAXCLASS + PAGE); { size_t x = lg_floor((psz<<1)-1); size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 2e2c11d8..2949de10 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -380,8 +380,6 @@ prof_thread_active_set prof_thread_name_get prof_thread_name_set psz2ind -psz2ind_clamp -psz2ind_impl psz2u rtree_child_read rtree_child_read_hard diff --git a/src/arena.c b/src/arena.c index fd3c5531..dd8e4d9c 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1696,7 +1696,7 @@ arena_new(tsdn_t *tsdn, unsigned ind) WITNESS_RANK_ARENA_LARGE)) return (NULL); - for (i = 0; i < NPSIZES; i++) { + for (i = 0; i < NPSIZES+1; i++) { extent_heap_new(&arena->extents_cached[i]); extent_heap_new(&arena->extents_retained[i]); } diff --git a/src/extent.c b/src/extent.c index ad78c879..a802ad90 100644 --- a/src/extent.c +++ b/src/extent.c @@ -45,7 +45,7 @@ static size_t highpages; */ static void extent_record(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES], + extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES+1], bool cache, extent_t *extent); /******************************************************************************/ @@ -190,11 +190,11 @@ extent_ad_comp(const extent_t *a, const extent_t *b) ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_ad_comp) static void -extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES], +extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1], extent_t *extent) { size_t psz = extent_size_quantize_floor(extent_size_get(extent)); - pszind_t pind = psz2ind_clamp(psz); + pszind_t pind = psz2ind(psz); malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx); @@ -202,11 +202,11 @@ extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES], } static void -extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES], +extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1], extent_t *extent) { size_t psz = extent_size_quantize_floor(extent_size_get(extent)); - pszind_t pind = psz2ind_clamp(psz); + pszind_t pind = psz2ind(psz); malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx); @@ -358,15 +358,14 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent) */ static extent_t * extent_first_best_fit(tsdn_t *tsdn, arena_t *arena, - extent_heap_t extent_heaps[NPSIZES], size_t size) + extent_heap_t extent_heaps[NPSIZES+1], size_t size) { pszind_t pind, i; malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); pind = psz2ind(extent_size_quantize_ceil(size)); - assert(pind < NPSIZES); - for (i = pind; i < NPSIZES; i++) { + for (i = pind; i < NPSIZES+1; i++) { extent_t *extent = extent_heap_first(&extent_heaps[i]); if (extent != NULL) return (extent); @@ -393,7 +392,7 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, static extent_t * extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - extent_heap_t extent_heaps[NPSIZES], bool locked, bool cache, + extent_heap_t extent_heaps[NPSIZES+1], bool locked, bool cache, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) { @@ -758,7 +757,7 @@ extent_can_coalesce(const extent_t *a, const extent_t *b) static void extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, - extent_heap_t extent_heaps[NPSIZES], bool cache) + extent_heap_t extent_heaps[NPSIZES+1], bool cache) { if (!extent_can_coalesce(a, b)) @@ -786,7 +785,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, static void extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent) + extent_heap_t extent_heaps[NPSIZES+1], bool cache, extent_t *extent) { extent_t *prev, *next; rtree_ctx_t rtree_ctx_fallback; diff --git a/src/jemalloc.c b/src/jemalloc.c index 86030172..dc0add49 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -83,7 +83,7 @@ enum { static uint8_t malloc_slow_flags; JEMALLOC_ALIGNED(CACHELINE) -const size_t pind2sz_tab[NPSIZES] = { +const size_t pind2sz_tab[NPSIZES+1] = { #define PSZ_yes(lg_grp, ndelta, lg_delta) \ (((ZU(1)<