diff --git a/include/jemalloc/internal/sz.h b/include/jemalloc/internal/sz.h index 68e558ab..6df541f6 100644 --- a/include/jemalloc/internal/sz.h +++ b/include/jemalloc/internal/sz.h @@ -315,4 +315,7 @@ sz_sa2u(size_t size, size_t alignment) { return usize; } +size_t sz_psz_quantize_floor(size_t size); +size_t sz_psz_quantize_ceil(size_t size); + #endif /* JEMALLOC_INTERNAL_SIZE_H */ diff --git a/src/extent.c b/src/extent.c index cdbf9098..af23ca29 100644 --- a/src/extent.c +++ b/src/extent.c @@ -249,59 +249,6 @@ extent_hooks_assure_initialized(arena_t *arena, } } -#ifndef JEMALLOC_JET -static -#endif -size_t -extent_size_quantize_floor(size_t size) { - size_t ret; - pszind_t pind; - - assert(size > 0); - assert((size & PAGE_MASK) == 0); - - pind = sz_psz2ind(size - sz_large_pad + 1); - if (pind == 0) { - /* - * Avoid underflow. This short-circuit would also do the right - * thing for all sizes in the range for which there are - * PAGE-spaced size classes, but it's simplest to just handle - * the one case that would cause erroneous results. - */ - return size; - } - ret = sz_pind2sz(pind - 1) + sz_large_pad; - assert(ret <= size); - return ret; -} - -#ifndef JEMALLOC_JET -static -#endif -size_t -extent_size_quantize_ceil(size_t size) { - size_t ret; - - assert(size > 0); - assert(size - sz_large_pad <= SC_LARGE_MAXCLASS); - assert((size & PAGE_MASK) == 0); - - ret = extent_size_quantize_floor(size); - if (ret < size) { - /* - * Skip a quantization that may have an adequately large extent, - * because under-sized extents may be mixed in. This only - * happens when an unusual size is requested, i.e. for aligned - * allocation, and is just one of several places where linear - * search would potentially find sufficiently aligned available - * memory somewhere lower. - */ - ret = sz_pind2sz(sz_psz2ind(ret - sz_large_pad + 1)) + - sz_large_pad; - } - return ret; -} - /* Generate pairing heap functions. */ ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp) @@ -342,7 +289,7 @@ extents_insert_locked(tsdn_t *tsdn, eset_t *eset, extent_t *extent) { assert(extent_state_get(extent) == eset->state); size_t size = extent_size_get(extent); - size_t psz = extent_size_quantize_floor(size); + size_t psz = sz_psz_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); if (extent_heap_empty(&eset->heaps[pind])) { bitmap_unset(eset->bitmap, &eset_bitmap_info, @@ -373,7 +320,7 @@ extents_remove_locked(tsdn_t *tsdn, eset_t *eset, extent_t *extent) { assert(extent_state_get(extent) == eset->state); size_t size = extent_size_get(extent); - size_t psz = extent_size_quantize_floor(size); + size_t psz = sz_psz_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); extent_heap_remove(&eset->heaps[pind], extent); @@ -405,8 +352,8 @@ extents_remove_locked(tsdn_t *tsdn, eset_t *eset, extent_t *extent) { static extent_t * extents_fit_alignment(eset_t *eset, size_t min_size, size_t max_size, size_t alignment) { - pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(min_size)); - pszind_t pind_max = sz_psz2ind(extent_size_quantize_ceil(max_size)); + pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size)); + pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size)); for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info, (size_t)pind); i < pind_max; i = @@ -444,7 +391,7 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset, size_t size) { extent_t *ret = NULL; - pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); + pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(size)); if (!maps_coalesce && !opt_retain) { /* diff --git a/src/sz.c b/src/sz.c index 8633fb05..7734f394 100644 --- a/src/sz.c +++ b/src/sz.c @@ -4,6 +4,54 @@ JEMALLOC_ALIGNED(CACHELINE) size_t sz_pind2sz_tab[SC_NPSIZES+1]; + +size_t +sz_psz_quantize_floor(size_t size) { + size_t ret; + pszind_t pind; + + assert(size > 0); + assert((size & PAGE_MASK) == 0); + + pind = sz_psz2ind(size - sz_large_pad + 1); + if (pind == 0) { + /* + * Avoid underflow. This short-circuit would also do the right + * thing for all sizes in the range for which there are + * PAGE-spaced size classes, but it's simplest to just handle + * the one case that would cause erroneous results. + */ + return size; + } + ret = sz_pind2sz(pind - 1) + sz_large_pad; + assert(ret <= size); + return ret; +} + +size_t +sz_psz_quantize_ceil(size_t size) { + size_t ret; + + assert(size > 0); + assert(size - sz_large_pad <= SC_LARGE_MAXCLASS); + assert((size & PAGE_MASK) == 0); + + ret = sz_psz_quantize_floor(size); + if (ret < size) { + /* + * Skip a quantization that may have an adequately large extent, + * because under-sized extents may be mixed in. This only + * happens when an unusual size is requested, i.e. for aligned + * allocation, and is just one of several places where linear + * search would potentially find sufficiently aligned available + * memory somewhere lower. + */ + ret = sz_pind2sz(sz_psz2ind(ret - sz_large_pad + 1)) + + sz_large_pad; + } + return ret; +} + static void sz_boot_pind2sz_tab(const sc_data_t *sc_data) { int pind = 0; diff --git a/test/unit/extent_quantize.c b/test/unit/extent_quantize.c index 0ca7a75d..64b3baa7 100644 --- a/test/unit/extent_quantize.c +++ b/test/unit/extent_quantize.c @@ -23,11 +23,11 @@ TEST_BEGIN(test_small_extent_size) { assert_d_eq(mallctlbymib(mib, miblen, (void *)&extent_size, &sz, NULL, 0), 0, "Unexpected mallctlbymib failure"); assert_zu_eq(extent_size, - extent_size_quantize_floor(extent_size), + sz_psz_quantize_floor(extent_size), "Small extent quantization should be a no-op " "(extent_size=%zu)", extent_size); assert_zu_eq(extent_size, - extent_size_quantize_ceil(extent_size), + sz_psz_quantize_ceil(extent_size), "Small extent quantization should be a no-op " "(extent_size=%zu)", extent_size); } @@ -65,8 +65,8 @@ TEST_BEGIN(test_large_extent_size) { &sz, NULL, 0), 0, "Unexpected mallctlbymib failure"); extent_size = cache_oblivious ? lextent_size + PAGE : lextent_size; - floor = extent_size_quantize_floor(extent_size); - ceil = extent_size_quantize_ceil(extent_size); + floor = sz_psz_quantize_floor(extent_size); + ceil = sz_psz_quantize_ceil(extent_size); assert_zu_eq(extent_size, floor, "Extent quantization should be a no-op for precise size " @@ -79,7 +79,7 @@ TEST_BEGIN(test_large_extent_size) { if (i > 0) { assert_zu_eq(extent_size_prev, - extent_size_quantize_floor(extent_size - PAGE), + sz_psz_quantize_floor(extent_size - PAGE), "Floor should be a precise size"); if (extent_size_prev < ceil_prev) { assert_zu_eq(ceil_prev, extent_size, @@ -91,7 +91,7 @@ TEST_BEGIN(test_large_extent_size) { } if (i + 1 < nlextents) { extent_size_prev = floor; - ceil_prev = extent_size_quantize_ceil(extent_size + + ceil_prev = sz_psz_quantize_ceil(extent_size + PAGE); } } @@ -109,8 +109,8 @@ TEST_BEGIN(test_monotonic) { size_t extent_size, floor, ceil; extent_size = i << LG_PAGE; - floor = extent_size_quantize_floor(extent_size); - ceil = extent_size_quantize_ceil(extent_size); + floor = sz_psz_quantize_floor(extent_size); + ceil = sz_psz_quantize_ceil(extent_size); assert_zu_le(floor, extent_size, "Floor should be <= (floor=%zu, extent_size=%zu, ceil=%zu)",