diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index bb65c7a9..11863fc7 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -569,7 +569,7 @@ unsigned arena_nthreads_get(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); arena_t *arena_new(tsdn_t *tsdn, unsigned ind); -bool arena_boot(void); +void arena_boot(void); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index d1b1db1e..440953ad 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -158,7 +158,6 @@ size_classes() { lg_tiny_maxclass='"NA"' nbins=0 npsizes=0 - slab_maxpgs=0 # Tiny size classes. ndelta=0 @@ -175,9 +174,6 @@ size_classes() { fi if [ ${bin} != "no" ] ; then nbins=$((${index} + 1)) - if [ ${pgs} -gt ${slab_maxpgs} ] ; then - slab_maxpgs=${pgs} - fi fi ntbins=$((${ntbins} + 1)) lg_tiny_maxclass=${lg_grp} # Final written value is correct. @@ -200,9 +196,6 @@ size_classes() { if [ ${psz} = "yes" ] ; then npsizes=$((${npsizes} + 1)) fi - if [ ${pgs} -gt ${slab_maxpgs} ] ; then - slab_maxpgs=${pgs} - fi fi while [ ${ndelta} -lt ${g} ] ; do size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} @@ -211,9 +204,6 @@ size_classes() { if [ ${psz} = "yes" ] ; then npsizes=$((${npsizes} + 1)) fi - if [ ${pgs} -gt ${slab_maxpgs} ] ; then - slab_maxpgs=${pgs} - fi done # All remaining groups. @@ -240,9 +230,6 @@ size_classes() { nbins=$((${index} + 1)) # Final written value is correct: small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" - if [ ${pgs} -gt ${slab_maxpgs} ] ; then - slab_maxpgs=${pgs} - fi if [ ${lg_g} -gt 0 ] ; then lg_large_minclass=$((${lg_grp} + 1)) else @@ -269,7 +256,6 @@ size_classes() { # - lg_tiny_maxclass # - lookup_maxclass # - small_maxclass - # - slab_maxpgs # - lg_large_minclass # - huge_maxclass } @@ -303,7 +289,6 @@ cat <> - LG_PAGE]) - return (size); - - /* - * Round down to the nearest run size that can actually be requested - * during normal large allocation. Add large_pad so that cache index - * randomization can offset the allocation from the page boundary. - */ - qsize = index2size(size2index(size - large_pad + 1) - 1) + large_pad; - if (qsize <= SMALL_MAXCLASS + large_pad) - return (run_quantize_floor_compute(size - large_pad)); - assert(qsize <= size); - return (qsize); -} - -static size_t -run_quantize_ceil_compute_hard(size_t size) -{ - size_t large_run_size_next; - - assert(size != 0); - assert(size == PAGE_CEILING(size)); - - /* - * Return the next quantized size greater than the input size. - * Quantized sizes comprise the union of run sizes that back small - * region runs, and run sizes that back large regions with no explicit - * alignment constraints. - */ - - if (size > SMALL_MAXCLASS) { - large_run_size_next = PAGE_CEILING(index2size(size2index(size - - large_pad) + 1) + large_pad); - } else - large_run_size_next = SIZE_T_MAX; - if ((size >> LG_PAGE) >= ZU(SLAB_MAXPGS)) - return (large_run_size_next); - - while (true) { - size += PAGE; - assert(size <= (ZU(SLAB_MAXPGS) << LG_PAGE)); - if (small_run_tab[size >> LG_PAGE]) { - if (large_run_size_next < size) - return (large_run_size_next); - return (size); - } - } -} - -static size_t -run_quantize_ceil_compute(size_t size) -{ - size_t qsize = run_quantize_floor_compute(size); - - if (qsize < size) { - /* - * Skip a quantization that may have an adequately large run, - * because under-sized runs may be mixed in. This only happens - * when an unusual size is requested, i.e. for aligned - * allocation, and is just one of several places where linear - * search would potentially find sufficiently aligned available - * memory somewhere lower. - */ - qsize = run_quantize_ceil_compute_hard(qsize); - } - return (qsize); -} - #ifdef JEMALLOC_JET #undef run_quantize_floor #define run_quantize_floor JEMALLOC_N(n_run_quantize_floor) @@ -172,13 +91,27 @@ static size_t run_quantize_floor(size_t size) { size_t ret; + pszind_t pind; assert(size > 0); assert(size <= HUGE_MAXCLASS); assert((size & PAGE_MASK) == 0); - ret = run_quantize_floor_tab[(size >> LG_PAGE) - 1]; - assert(ret == run_quantize_floor_compute(size)); + assert(size != 0); + assert(size == PAGE_CEILING(size)); + + pind = psz2ind(size - large_pad + 1); + if (pind == 0) { + /* + * Avoid underflow. This short-circuit would also do the right + * thing for all sizes in the range for which there are + * PAGE-spaced size classes, but it's simplest to just handle + * the one case that would cause erroneous results. + */ + return (size); + } + ret = pind2sz(pind - 1) + large_pad; + assert(ret <= size); return (ret); } #ifdef JEMALLOC_JET @@ -200,8 +133,18 @@ run_quantize_ceil(size_t size) assert(size <= HUGE_MAXCLASS); assert((size & PAGE_MASK) == 0); - ret = run_quantize_ceil_tab[(size >> LG_PAGE) - 1]; - assert(ret == run_quantize_ceil_compute(size)); + ret = run_quantize_floor(size); + if (ret < size) { + /* + * Skip a quantization that may have an adequately large run, + * because under-sized runs may be mixed in. This only happens + * when an unusual size is requested, i.e. for aligned + * allocation, and is just one of several places where linear + * search would potentially find sufficiently aligned available + * memory somewhere lower. + */ + ret = pind2sz(psz2ind(ret - large_pad + 1)) + large_pad; + } return (ret); } #ifdef JEMALLOC_JET @@ -3483,64 +3426,7 @@ arena_new(tsdn_t *tsdn, unsigned ind) return (arena); } -static bool -small_run_size_init(void) -{ - - assert(SLAB_MAXPGS != 0); - - small_run_tab = (bool *)base_alloc(NULL, sizeof(bool) * SLAB_MAXPGS); - if (small_run_tab == NULL) - return (true); - -#define TAB_INIT_bin_yes(index, size) { \ - const arena_bin_info_t *bin_info = \ - &arena_bin_info[index]; \ - small_run_tab[bin_info->run_size >> LG_PAGE] = true; \ - } -#define TAB_INIT_bin_no(index, size) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, run_size, \ - lg_delta_lookup) \ - TAB_INIT_bin_##bin(index, (ZU(1)<> LG_PAGE)); - if (run_quantize_floor_tab == NULL) - return (true); - - run_quantize_ceil_tab = (size_t *)base_alloc(NULL, sizeof(size_t) * - (run_quantize_max >> LG_PAGE)); - if (run_quantize_ceil_tab == NULL) - return (true); - - for (i = 1; i <= run_quantize_max >> LG_PAGE; i++) { - size_t run_size = i << LG_PAGE; - - run_quantize_floor_tab[i-1] = - run_quantize_floor_compute(run_size); - run_quantize_ceil_tab[i-1] = - run_quantize_ceil_compute(run_size); - } - - return (false); -} - -bool +void arena_boot(void) { unsigned i; @@ -3586,13 +3472,6 @@ arena_boot(void) assert(large_maxclass > 0); nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS); nhclasses = NSIZES - nlclasses - NBINS; - - if (small_run_size_init()) - return (true); - if (run_quantize_init()) - return (true); - - return (false); } void diff --git a/src/jemalloc.c b/src/jemalloc.c index 849e9418..929f3b87 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1234,8 +1234,7 @@ malloc_init_hard_a0_locked() return (true); if (config_prof) prof_boot1(); - if (arena_boot()) - return (true); + arena_boot(); if (config_tcache && tcache_boot(TSDN_NULL)) return (true); if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS))