From 6b197fdd460be8bf3379da91d42e677dd5b5437a Mon Sep 17 00:00:00 2001 From: guangli-dai Date: Tue, 22 Aug 2023 16:31:54 -0700 Subject: [PATCH] Pre-generate ncached_max for all bins for better tcache_max tuning experience. --- include/jemalloc/internal/arena_inlines_b.h | 8 +- include/jemalloc/internal/cache_bin.h | 91 +++++--- include/jemalloc/internal/tcache_externs.h | 7 +- include/jemalloc/internal/tcache_inlines.h | 144 +++++------- include/jemalloc/internal/tcache_structs.h | 4 +- src/arena.c | 22 +- src/cache_bin.c | 20 +- src/ctl.c | 6 +- src/jemalloc.c | 3 +- src/tcache.c | 244 +++++++++++++------- test/unit/cache_bin.c | 132 ++++++----- test/unit/tcache_max.c | 44 ++-- 12 files changed, 417 insertions(+), 308 deletions(-) diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index c4d1c887..a4bacd8b 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -198,7 +198,9 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, assert(sz_can_use_slab(size)); return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache, size, ind, zero, slow_path); - } else if (likely(size <= tcache_max_get(tcache))) { + } else if (likely(ind < TCACHE_NBINS_MAX && + !tcache_bin_disabled(ind, &tcache->bins[ind], + tcache->tcache_slow))) { return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache, size, ind, zero, slow_path); } @@ -298,7 +300,9 @@ JEMALLOC_ALWAYS_INLINE void arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind, bool slow_path) { assert (!tsdn_null(tsdn) && tcache != NULL); - if (szind < tcache_nhbins_get(tcache)) { + if (szind < TCACHE_NBINS_MAX && + !tcache_bin_disabled(szind, &tcache->bins[szind], + tcache->tcache_slow)) { if (config_prof && unlikely(szind < SC_NBINS)) { arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); } else { diff --git a/include/jemalloc/internal/cache_bin.h b/include/jemalloc/internal/cache_bin.h index 78ac3295..2c831caf 100644 --- a/include/jemalloc/internal/cache_bin.h +++ b/include/jemalloc/internal/cache_bin.h @@ -23,16 +23,20 @@ */ typedef uint16_t cache_bin_sz_t; +#define JUNK_ADDR ((uintptr_t)0x7a7a7a7a7a7a7a7aULL) /* * Leave a noticeable mark pattern on the cache bin stack boundaries, in case a * bug starts leaking those. Make it look like the junk pattern but be distinct * from it. */ -static const uintptr_t cache_bin_preceding_junk = - (uintptr_t)0x7a7a7a7a7a7a7a7aULL; -/* Note: a7 vs. 7a above -- this tells you which pointer leaked. */ -static const uintptr_t cache_bin_trailing_junk = - (uintptr_t)0xa7a7a7a7a7a7a7a7ULL; +static const uintptr_t cache_bin_preceding_junk = JUNK_ADDR; +/* Note: JUNK_ADDR vs. JUNK_ADDR + 1 -- this tells you which pointer leaked. */ +static const uintptr_t cache_bin_trailing_junk = JUNK_ADDR + 1; +/* + * A pointer used to initialize a fake stack_head for disabled small bins + * so that the enabled/disabled assessment does not rely on ncached_max. + */ +extern const uintptr_t disabled_bin; /* * That implies the following value, for the maximum number of items in any @@ -174,9 +178,35 @@ cache_bin_nonfast_aligned(const void *ptr) { return ((uintptr_t)ptr & san_cache_bin_nonfast_mask) == 0; } +static inline const void * +cache_bin_disabled_bin_stack(void) { + return &disabled_bin; +} + +/* + * If a cache bin was zero initialized (either because it lives in static or + * thread-local storage, or was memset to 0), this function indicates whether or + * not cache_bin_init was called on it. + */ +static inline bool +cache_bin_still_zero_initialized(cache_bin_t *bin) { + return bin->stack_head == NULL; +} + +static inline bool +cache_bin_disabled(cache_bin_t *bin) { + bool disabled = (bin->stack_head == cache_bin_disabled_bin_stack()); + if (disabled) { + assert((uintptr_t)(*bin->stack_head) == JUNK_ADDR); + } + return disabled; +} + /* Returns ncached_max: Upper limit on ncached. */ static inline cache_bin_sz_t -cache_bin_info_ncached_max(cache_bin_info_t *info) { +cache_bin_info_ncached_max_get(cache_bin_t *bin, cache_bin_info_t *info) { + assert(!cache_bin_disabled(bin)); + assert(info == &bin->bin_info); return info->ncached_max; } @@ -234,7 +264,7 @@ cache_bin_ncached_get_internal(cache_bin_t *bin) { static inline cache_bin_sz_t cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_sz_t n = cache_bin_ncached_get_internal(bin); - assert(n <= cache_bin_info_ncached_max(info)); + assert(n <= cache_bin_info_ncached_max_get(bin, info)); return n; } @@ -271,7 +301,7 @@ cache_bin_empty_position_get(cache_bin_t *bin) { static inline uint16_t cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { return (uint16_t)bin->low_bits_empty - - info->ncached_max * sizeof(void *); + cache_bin_info_ncached_max_get(bin, info) * sizeof(void *); } /* @@ -281,7 +311,7 @@ cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { */ static inline void ** cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { - cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(info); + cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info); void **ret = cache_bin_empty_position_get(bin) - ncached_max; assert(ret <= bin->stack_head); @@ -313,7 +343,7 @@ cache_bin_low_water_get_internal(cache_bin_t *bin) { static inline cache_bin_sz_t cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_sz_t low_water = cache_bin_low_water_get_internal(bin); - assert(low_water <= cache_bin_info_ncached_max(info)); + assert(low_water <= cache_bin_info_ncached_max_get(bin, info)); assert(low_water <= cache_bin_ncached_get_local(bin, info)); cache_bin_assert_earlier(bin, (uint16_t)(uintptr_t)bin->stack_head, @@ -328,11 +358,13 @@ cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) { */ static inline void cache_bin_low_water_set(cache_bin_t *bin) { + assert(!cache_bin_disabled(bin)); bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head; } static inline void cache_bin_low_water_adjust(cache_bin_t *bin) { + assert(!cache_bin_disabled(bin)); if (cache_bin_ncached_get_internal(bin) < cache_bin_low_water_get_internal(bin)) { cache_bin_low_water_set(bin); @@ -494,25 +526,26 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) { /* Get the number of stashed pointers. */ JEMALLOC_ALWAYS_INLINE cache_bin_sz_t cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) { - cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(info); + cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info); uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin, info); cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound, bin->low_bits_full) / sizeof(void *); assert(n <= ncached_max); + if (config_debug && n != 0) { + /* Below are for assertions only. */ + void **low_bound = cache_bin_low_bound_get(bin, info); - /* Below are for assertions only. */ - void **low_bound = cache_bin_low_bound_get(bin, info); - - assert((uint16_t)(uintptr_t)low_bound == low_bits_low_bound); - void *stashed = *(low_bound + n - 1); - bool aligned = cache_bin_nonfast_aligned(stashed); + assert((uint16_t)(uintptr_t)low_bound == low_bits_low_bound); + void *stashed = *(low_bound + n - 1); + bool aligned = cache_bin_nonfast_aligned(stashed); #ifdef JEMALLOC_JET - /* Allow arbitrary pointers to be stashed in tests. */ - aligned = true; + /* Allow arbitrary pointers to be stashed in tests. */ + aligned = true; #endif - assert(n == 0 || (stashed != NULL && aligned)); + assert(stashed != NULL && aligned); + } return n; } @@ -520,7 +553,7 @@ cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) { JEMALLOC_ALWAYS_INLINE cache_bin_sz_t cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin, info); - assert(n <= cache_bin_info_ncached_max(info)); + assert(n <= cache_bin_info_ncached_max_get(bin, info)); return n; } @@ -541,8 +574,8 @@ cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) { * This function should not call other utility functions because the racy * condition may cause unexpected / undefined behaviors in unverified utility * functions. Currently, this function calls two utility functions - * cache_bin_info_ncached_max and cache_bin_low_bits_low_bound_get because they - * help access values that will not be concurrently modified. + * cache_bin_info_ncached_max_get and cache_bin_low_bits_low_bound_get because + * they help access values that will not be concurrently modified. */ static inline void cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info, @@ -552,7 +585,8 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info, (uint16_t)(uintptr_t)bin->stack_head; cache_bin_sz_t n = diff / sizeof(void *); - assert(n <= cache_bin_info_ncached_max(info)); + cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info); + assert(n <= ncached_max); *ncached = n; /* Racy version of cache_bin_nstashed_get_internal. */ @@ -560,7 +594,7 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info, info); n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *); - assert(n <= cache_bin_info_ncached_max(info)); + assert(n <= ncached_max); *nstashed = n; /* Note that cannot assert ncached + nstashed <= ncached_max (racy). */ } @@ -697,13 +731,8 @@ void cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void cache_bin_postincrement(void *alloc, size_t *cur_offset); void cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, size_t *cur_offset); +void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max); -/* - * If a cache bin was zero initialized (either because it lives in static or - * thread-local storage, or was memset to 0), this function indicates whether or - * not cache_bin_init was called on it. - */ -bool cache_bin_still_zero_initialized(cache_bin_t *bin); bool cache_bin_stack_use_thp(void); #endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */ diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index 87d243a1..8ca966d7 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -24,9 +24,9 @@ extern unsigned opt_lg_tcache_flush_large_div; * large-object bins. This is only used during threads initialization and * changing it will not reflect on initialized threads as expected. Thus, * it should not be changed on the fly. To change the number of tcache bins - * in use, refer to tcache_nhbins of each tcache. + * in use, refer to tcache_nbins of each tcache. */ -extern unsigned global_do_not_change_nhbins; +extern unsigned global_do_not_change_nbins; /* * Maximum cached size class. Same as above, this is only used during threads @@ -58,6 +58,7 @@ void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, void tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena); tcache_t *tcache_create_explicit(tsd_t *tsd); +void thread_tcache_max_set(tsd_t *tsd, size_t tcache_max); void tcache_cleanup(tsd_t *tsd); void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind); @@ -70,8 +71,8 @@ void tcache_prefork(tsdn_t *tsdn); void tcache_postfork_parent(tsdn_t *tsdn); void tcache_postfork_child(tsdn_t *tsdn); void tcache_flush(tsd_t *tsd); -bool tsd_tcache_data_init(tsd_t *tsd, arena_t *arena); bool tsd_tcache_enabled_data_init(tsd_t *tsd); +void tcache_enabled_set(tsd_t *tsd, bool enabled); void tcache_assert_initialized(tcache_t *tcache); diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index 97501ee2..68481113 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -18,94 +18,72 @@ tcache_enabled_get(tsd_t *tsd) { return tsd_tcache_enabled_get(tsd); } -static inline void -tcache_enabled_set(tsd_t *tsd, bool enabled) { - bool was_enabled = tsd_tcache_enabled_get(tsd); - - if (!was_enabled && enabled) { - tsd_tcache_data_init(tsd, NULL); - } else if (was_enabled && !enabled) { - tcache_cleanup(tsd); - } - /* Commit the state last. Above calls check current state. */ - tsd_tcache_enabled_set(tsd, enabled); - tsd_slow_update(tsd); -} - static inline unsigned -tcache_nhbins_get(tcache_t *tcache) { - assert(tcache != NULL); - assert(tcache->tcache_nhbins <= TCACHE_NBINS_MAX); - return tcache->tcache_nhbins; +tcache_nbins_get(tcache_slow_t *tcache_slow) { + assert(tcache_slow != NULL); + unsigned nbins = tcache_slow->tcache_nbins; + assert(nbins <= TCACHE_NBINS_MAX); + return nbins; } static inline size_t -tcache_max_get(tcache_t *tcache) { - assert(tcache != NULL); - assert(tcache->tcache_max <= TCACHE_MAXCLASS_LIMIT); - return tcache->tcache_max; +tcache_max_get(tcache_slow_t *tcache_slow) { + assert(tcache_slow != NULL); + size_t tcache_max = sz_index2size(tcache_nbins_get(tcache_slow) - 1); + assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); + return tcache_max; } static inline void -tcache_max_and_nhbins_set(tcache_t *tcache, size_t tcache_max) { - assert(tcache != NULL); +tcache_max_set(tcache_slow_t *tcache_slow, size_t tcache_max) { + assert(tcache_slow != NULL); assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); - tcache->tcache_max = tcache_max; - tcache->tcache_nhbins = sz_size2index(tcache_max) + 1; + tcache_slow->tcache_nbins = sz_size2index(tcache_max) + 1; } static inline void -thread_tcache_max_and_nhbins_set(tsd_t *tsd, size_t tcache_max) { - assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); - assert(tcache_max == sz_s2u(tcache_max)); - tcache_t *tcache = tsd_tcachep_get(tsd); - tcache_slow_t *tcache_slow; - assert(tcache != NULL); - - bool enabled = tcache_available(tsd); - arena_t *assigned_arena; - if (enabled) { - tcache_slow = tcache_slow_get(tsd); - assert(tcache != NULL && tcache_slow != NULL); - assigned_arena = tcache_slow->arena; - /* Shutdown and reboot the tcache for a clean slate. */ - tcache_cleanup(tsd); +tcache_bin_settings_backup(tcache_t *tcache, + cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { + for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) { + cache_bin_info_init(&tcache_bin_info[i], + tcache->bins[i].bin_info.ncached_max); } +} + +JEMALLOC_ALWAYS_INLINE bool +tcache_bin_disabled(szind_t ind, cache_bin_t *bin, + tcache_slow_t *tcache_slow) { + assert(bin != NULL); + bool disabled = cache_bin_disabled(bin); /* - * Still set tcache_max and tcache_nhbins of the tcache even if - * the tcache is not available yet because the values are - * stored in tsd_t and are always available for changing. - */ - tcache_max_and_nhbins_set(tcache, tcache_max); - - if (enabled) { - tsd_tcache_data_init(tsd, assigned_arena); + * If a bin's ind >= nbins or ncached_max == 0, it must be disabled. + * However, when ind < nbins, it could be either enabled + * (ncached_max > 0) or disabled (ncached_max == 0). Similarly, when + * ncached_max > 0, it could be either enabled (ind < nbins) or + * disabled (ind >= nbins). Thus, if a bin is disabled, it has either + * ind >= nbins or ncached_max == 0. If a bin is enabled, it has + * ind < nbins and ncached_max > 0. + */ + unsigned nbins = tcache_nbins_get(tcache_slow); + cache_bin_sz_t ncached_max = bin->bin_info.ncached_max; + if (ind >= nbins) { + assert(disabled); + } else { + assert(!disabled || ncached_max == 0); + } + if (ncached_max == 0) { + assert(disabled); + } else { + assert(!disabled || ind >= nbins); + } + if (disabled) { + assert(ind >= nbins || ncached_max == 0); + } else { + assert(ind < nbins && ncached_max > 0); } - assert(tcache_nhbins_get(tcache) == sz_size2index(tcache_max) + 1); -} - -JEMALLOC_ALWAYS_INLINE bool -tcache_small_bin_disabled(szind_t ind, cache_bin_t *bin) { - assert(ind < SC_NBINS); - assert(bin != NULL); - bool ret = cache_bin_info_ncached_max(&bin->bin_info) == 0; - if (ret) { - /* small size class but cache bin disabled. */ - assert((uintptr_t)(*bin->stack_head) == - cache_bin_preceding_junk); - } - - return ret; -} - -JEMALLOC_ALWAYS_INLINE bool -tcache_large_bin_disabled(szind_t ind, cache_bin_t *bin) { - assert(ind >= SC_NBINS); - assert(bin != NULL); - return (cache_bin_info_ncached_max(&bin->bin_info) == 0 || - cache_bin_still_zero_initialized(bin)); + return disabled; } JEMALLOC_ALWAYS_INLINE void * @@ -124,7 +102,8 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, if (unlikely(arena == NULL)) { return NULL; } - if (unlikely(tcache_small_bin_disabled(binind, bin))) { + if (unlikely(tcache_bin_disabled(binind, bin, + tcache->tcache_slow))) { /* stats and zero are handled directly by the arena. */ return arena_malloc_hard(tsd_tsdn(tsd), arena, size, binind, zero, /* slab */ true); @@ -157,8 +136,9 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, void *ret; bool tcache_success; - assert(binind >= SC_NBINS && binind < tcache_nhbins_get(tcache)); cache_bin_t *bin = &tcache->bins[binind]; + assert(binind >= SC_NBINS && + !tcache_bin_disabled(binind, bin, tcache->tcache_slow)); ret = cache_bin_alloc(bin, &tcache_success); assert(tcache_success == (ret != NULL)); if (unlikely(!tcache_success)) { @@ -180,7 +160,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, } else { if (unlikely(zero)) { size_t usize = sz_index2size(binind); - assert(usize <= tcache_max_get(tcache)); + assert(usize <= tcache_max_get(tcache->tcache_slow)); memset(ret, 0, usize); } @@ -214,12 +194,13 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, } if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { - if (unlikely(tcache_small_bin_disabled(binind, bin))) { + if (unlikely(tcache_bin_disabled(binind, bin, + tcache->tcache_slow))) { arena_dalloc_small(tsd_tsdn(tsd), ptr); return; } - cache_bin_sz_t max = cache_bin_info_ncached_max( - &bin->bin_info); + cache_bin_sz_t max = cache_bin_info_ncached_max_get( + bin, &bin->bin_info); unsigned remain = max >> opt_lg_tcache_flush_small_div; tcache_bin_flush_small(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); @@ -232,12 +213,13 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) { assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SC_SMALL_MAXCLASS); - assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_max_get(tcache)); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= + tcache_max_get(tcache->tcache_slow)); cache_bin_t *bin = &tcache->bins[binind]; if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { - unsigned remain = cache_bin_info_ncached_max( - &bin->bin_info) >> opt_lg_tcache_flush_large_div; + unsigned remain = cache_bin_info_ncached_max_get( + bin, &bin->bin_info) >> opt_lg_tcache_flush_large_div; tcache_bin_flush_large(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); diff --git a/include/jemalloc/internal/tcache_structs.h b/include/jemalloc/internal/tcache_structs.h index b51e10a7..d94099b0 100644 --- a/include/jemalloc/internal/tcache_structs.h +++ b/include/jemalloc/internal/tcache_structs.h @@ -31,6 +31,8 @@ struct tcache_slow_s { /* The arena this tcache is associated with. */ arena_t *arena; + /* The number of bins activated in the tcache. */ + unsigned tcache_nbins; /* Next bin to GC. */ szind_t next_gc_bin; /* For small bins, fill (ncached_max >> lg_fill_div). */ @@ -55,8 +57,6 @@ struct tcache_slow_s { struct tcache_s { tcache_slow_t *tcache_slow; - unsigned tcache_nhbins; - size_t tcache_max; cache_bin_t bins[TCACHE_NBINS_MAX]; }; diff --git a/src/arena.c b/src/arena.c index d937c349..4a383670 100644 --- a/src/arena.c +++ b/src/arena.c @@ -163,17 +163,13 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) { for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) { cache_bin_t *cache_bin = &descriptor->bins[i]; + if (cache_bin_disabled(cache_bin)) { + continue; + } + cache_bin_sz_t ncached, nstashed; cache_bin_nitems_get_remote(cache_bin, &cache_bin->bin_info, &ncached, &nstashed); - - if ((i < SC_NBINS && - tcache_small_bin_disabled(i, cache_bin)) || - (i >= SC_NBINS && - tcache_large_bin_disabled(i, cache_bin))) { - assert(ncached == 0 && nstashed == 0); - } - astats->tcache_bytes += ncached * sz_index2size(i); astats->tcache_stashed_bytes += nstashed * sz_index2size(i); @@ -730,11 +726,13 @@ arena_dalloc_promoted_impl(tsdn_t *tsdn, void *ptr, tcache_t *tcache, */ safety_check_verify_redzone(ptr, usize, bumped_usize); } + szind_t bumped_ind = sz_size2index(bumped_usize); if (bumped_usize >= SC_LARGE_MINCLASS && - tcache != NULL && - bumped_usize <= tcache_max_get(tcache)) { - tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, - sz_size2index(bumped_usize), slow_path); + tcache != NULL && bumped_ind < TCACHE_NBINS_MAX && + !tcache_bin_disabled(bumped_ind, &tcache->bins[bumped_ind], + tcache->tcache_slow)) { + tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, bumped_ind, + slow_path); } else { large_dalloc(tsdn, edata); } diff --git a/src/cache_bin.c b/src/cache_bin.c index 2ad2062d..67b6327b 100644 --- a/src/cache_bin.c +++ b/src/cache_bin.c @@ -5,10 +5,11 @@ #include "jemalloc/internal/cache_bin.h" #include "jemalloc/internal/safety_check.h" +const uintptr_t disabled_bin = JUNK_ADDR; + void cache_bin_info_init(cache_bin_info_t *info, cache_bin_sz_t ncached_max) { - assert(ncached_max <= CACHE_BIN_NCACHED_MAX); size_t stack_size = (size_t)ncached_max * sizeof(void *); assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8))); info->ncached_max = (cache_bin_sz_t)ncached_max; @@ -37,7 +38,6 @@ cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, */ *size = sizeof(void *) * 2; for (szind_t i = 0; i < ninfos; i++) { - assert(infos[i].ncached_max > 0); *size += infos[i].ncached_max * sizeof(void *); } @@ -98,13 +98,21 @@ cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, cache_bin_sz_t free_spots = cache_bin_diff(bin, bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head); assert(free_spots == bin_stack_size); - assert(cache_bin_ncached_get_local(bin, info) == 0); + if (!cache_bin_disabled(bin)) { + assert(cache_bin_ncached_get_local(bin, &bin->bin_info) == 0); + } assert(cache_bin_empty_position_get(bin) == empty_position); assert(bin_stack_size > 0 || empty_position == full_position); } -bool -cache_bin_still_zero_initialized(cache_bin_t *bin) { - return bin->stack_head == NULL; +void +cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max) { + const void *fake_stack = cache_bin_disabled_bin_stack(); + size_t fake_offset = 0; + cache_bin_info_t fake_info; + cache_bin_info_init(&fake_info, 0); + cache_bin_init(bin, &fake_info, (void *)fake_stack, &fake_offset); + cache_bin_info_init(&bin->bin_info, ncached_max); + assert(fake_offset == 0); } diff --git a/src/ctl.c b/src/ctl.c index b0277c0a..af22d0aa 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -2317,7 +2317,7 @@ thread_tcache_max_ctl(tsd_t *tsd, const size_t *mib, /* pointer to tcache_t always exists even with tcache disabled. */ tcache_t *tcache = tsd_tcachep_get(tsd); assert(tcache != NULL); - oldval = tcache_max_get(tcache); + oldval = tcache_max_get(tcache->tcache_slow); READ(oldval, size_t); if (newp != NULL) { @@ -2332,7 +2332,7 @@ thread_tcache_max_ctl(tsd_t *tsd, const size_t *mib, } new_tcache_max = sz_s2u(new_tcache_max); if(new_tcache_max != oldval) { - thread_tcache_max_and_nhbins_set(tsd, new_tcache_max); + thread_tcache_max_set(tsd, new_tcache_max); } } @@ -3155,7 +3155,7 @@ CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_GEN(arenas_tcache_max, global_do_not_change_tcache_maxclass, size_t) CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned) -CTL_RO_NL_GEN(arenas_nhbins, global_do_not_change_nhbins, unsigned) +CTL_RO_NL_GEN(arenas_nhbins, global_do_not_change_nbins, unsigned) CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) diff --git a/src/jemalloc.c b/src/jemalloc.c index 7aa6a1cd..4bf5cbff 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -4140,7 +4140,8 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) { tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, /* slow */ true, /* is_alloc */ true); if (likely(tcache != NULL && - ind < tcache_nhbins_get(tcache)) && progress < batch) { + !tcache_bin_disabled(ind, &tcache->bins[ind], + tcache->tcache_slow)) && progress < batch) { if (bin == NULL) { bin = &tcache->bins[ind]; } diff --git a/src/tcache.c b/src/tcache.c index 2c0a7e2e..3070193c 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -60,10 +60,10 @@ unsigned opt_lg_tcache_flush_large_div = 1; /* * Number of cache bins enabled, including both large and small. This value - * is only used to initialize tcache_nhbins in the per-thread tcache. + * is only used to initialize tcache_nbins in the per-thread tcache. * Directly modifying it will not affect threads already launched. */ -unsigned global_do_not_change_nhbins; +unsigned global_do_not_change_nbins; /* * Max size class to be cached (can be small or large). This value is only used * to initialize tcache_max in the per-thread tcache. Directly modifying it @@ -129,6 +129,7 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, assert(szind < SC_NBINS); cache_bin_t *cache_bin = &tcache->bins[szind]; + assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow)); cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, &cache_bin->bin_info); cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, @@ -155,7 +156,7 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, * Reduce fill count by 2X. Limit lg_fill_div such that * the fill count is always at least 1. */ - if ((cache_bin_info_ncached_max(&cache_bin->bin_info) + if ((cache_bin_info_ncached_max_get(cache_bin, &cache_bin->bin_info) >> (tcache_slow->lg_fill_div[szind] + 1)) >= 1) { tcache_slow->lg_fill_div[szind]++; } @@ -167,6 +168,7 @@ tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, /* Like the small GC; flush 3/4 of untouched items. */ assert(szind >= SC_NBINS); cache_bin_t *cache_bin = &tcache->bins[szind]; + assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow)); cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, &cache_bin->bin_info); cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, @@ -187,8 +189,12 @@ tcache_event(tsd_t *tsd) { bool is_small = (szind < SC_NBINS); cache_bin_t *cache_bin = &tcache->bins[szind]; - tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small); + if (tcache_bin_disabled(szind, cache_bin, tcache_slow)) { + goto label_done; + } + tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, + is_small); cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, &cache_bin->bin_info); if (low_water > 0) { @@ -210,8 +216,9 @@ tcache_event(tsd_t *tsd) { } cache_bin_low_water_set(cache_bin); +label_done: tcache_slow->next_gc_bin++; - if (tcache_slow->next_gc_bin == tcache_nhbins_get(tcache)) { + if (tcache_slow->next_gc_bin == tcache_nbins_get(tcache_slow)) { tcache_slow->next_gc_bin = 0; } } @@ -236,8 +243,9 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, void *ret; assert(tcache_slow->arena != NULL); - unsigned nfill = cache_bin_info_ncached_max(&cache_bin->bin_info) - >> tcache_slow->lg_fill_div[binind]; + assert(!tcache_bin_disabled(binind, cache_bin, tcache_slow)); + unsigned nfill = cache_bin_info_ncached_max_get(cache_bin, + &cache_bin->bin_info) >> tcache_slow->lg_fill_div[binind]; arena_cache_bin_fill_small(tsdn, arena, cache_bin, &cache_bin->bin_info, binind, nfill); tcache_slow->bin_refilled[binind] = true; @@ -321,7 +329,7 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, if (small) { assert(binind < SC_NBINS); } else { - assert(binind < tcache_nhbins_get(tcache)); + assert(binind < tcache_nbins_get(tcache_slow)); } arena_t *tcache_arena = tcache_slow->arena; assert(tcache_arena != NULL); @@ -508,6 +516,7 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, JEMALLOC_ALWAYS_INLINE void tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, unsigned rem, bool small) { + assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small); cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, @@ -551,6 +560,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, bool is_small) { + assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); cache_bin_info_t *info = &cache_bin->bin_info; /* * The two below are for assertion only. The content of original cached @@ -562,7 +572,8 @@ tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, info); cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin, info); - assert(orig_cached + nstashed <= cache_bin_info_ncached_max(info)); + assert(orig_cached + nstashed <= + cache_bin_info_ncached_max_get(cache_bin, info)); if (nstashed == 0) { return; } @@ -637,33 +648,11 @@ tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, } static void -tcache_max_and_nhbins_init(tcache_t *tcache) { - assert(tcache != NULL); +tcache_default_settings_init(tcache_slow_t *tcache_slow) { + assert(tcache_slow != NULL); assert(global_do_not_change_tcache_maxclass != 0); - assert(global_do_not_change_nhbins != 0); - tcache->tcache_max = global_do_not_change_tcache_maxclass; - tcache->tcache_nhbins = global_do_not_change_nhbins; - assert(tcache->tcache_nhbins == sz_size2index(tcache->tcache_max) + 1); -} - -bool -tsd_tcache_enabled_data_init(tsd_t *tsd) { - /* Called upon tsd initialization. */ - tsd_tcache_enabled_set(tsd, opt_tcache); - /* - * tcache is not available yet, but we need to set up its tcache_max - * and tcache_nhbins in advance. - */ - tcache_t *tcache = tsd_tcachep_get(tsd); - tcache_max_and_nhbins_init(tcache); - tsd_slow_update(tsd); - - if (opt_tcache) { - /* Trigger tcache init. */ - tsd_tcache_data_init(tsd, NULL); - } - - return false; + assert(global_do_not_change_nbins != 0); + tcache_slow->tcache_nbins = global_do_not_change_nbins; } static void @@ -679,19 +668,15 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, /* * We reserve cache bins for all small size classes, even if some may - * not get used (i.e. bins higher than tcache_nhbins). This allows + * not get used (i.e. bins higher than tcache_nbins). This allows * the fast and common paths to access cache bin metadata safely w/o * worrying about which ones are disabled. */ - unsigned tcache_nhbins = tcache_nhbins_get(tcache); - unsigned n_reserved_bins = tcache_nhbins < SC_NBINS ? SC_NBINS - : tcache_nhbins; - memset(tcache->bins, 0, sizeof(cache_bin_t) * n_reserved_bins); - + unsigned tcache_nbins = tcache_nbins_get(tcache_slow); size_t cur_offset = 0; - cache_bin_preincrement(tcache_bin_info, tcache_nhbins, mem, + cache_bin_preincrement(tcache_bin_info, tcache_nbins, mem, &cur_offset); - for (unsigned i = 0; i < tcache_nhbins; i++) { + for (unsigned i = 0; i < tcache_nbins; i++) { if (i < SC_NBINS) { tcache_slow->lg_fill_div[i] = 1; tcache_slow->bin_refilled[i] = false; @@ -699,40 +684,40 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, = tcache_gc_item_delay_compute(i); } cache_bin_t *cache_bin = &tcache->bins[i]; - cache_bin_init(cache_bin, &tcache_bin_info[i], mem, - &cur_offset); + if (tcache_bin_info[i].ncached_max > 0) { + cache_bin_init(cache_bin, &tcache_bin_info[i], mem, + &cur_offset); + } else { + cache_bin_init_disabled(cache_bin, + tcache_bin_info[i].ncached_max); + } } /* - * For small size classes beyond tcache_max(i.e. - * tcache_nhbins< NBINS), their cache bins are initialized to a state - * to safely and efficiently fail all fastpath alloc / free, so that - * no additional check around tcache_nhbins is needed on fastpath. + * Initialize all disabled bins to a state that can safely and + * efficiently fail all fastpath alloc / free, so that no additional + * check around tcache_nbins is needed on fastpath. Yet we still + * store the ncached_max in the bin_info for future usage. */ - for (unsigned i = tcache_nhbins; i < SC_NBINS; i++) { - /* Disabled small bins. */ + for (unsigned i = tcache_nbins; i < TCACHE_NBINS_MAX; i++) { cache_bin_t *cache_bin = &tcache->bins[i]; - void *fake_stack = mem; - size_t fake_offset = 0; - - cache_bin_init(cache_bin, &tcache_bin_info[i], fake_stack, - &fake_offset); - assert(tcache_small_bin_disabled(i, cache_bin)); + cache_bin_init_disabled(cache_bin, + tcache_bin_info[i].ncached_max); + assert(tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)); } cache_bin_postincrement(mem, &cur_offset); if (config_debug) { /* Sanity check that the whole stack is used. */ size_t size, alignment; - cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins, + cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins, &size, &alignment); assert(cur_offset == size); } } static inline unsigned -tcache_ncached_max_compute(szind_t szind, unsigned current_nhbins) { +tcache_ncached_max_compute(szind_t szind) { if (szind >= SC_NBINS) { - assert(szind < current_nhbins); return opt_tcache_nslots_large; } unsigned slab_nregs = bin_infos[szind].nregs; @@ -788,32 +773,28 @@ tcache_ncached_max_compute(szind_t szind, unsigned current_nhbins) { } static void -tcache_bin_info_compute(cache_bin_info_t *tcache_bin_info, - unsigned tcache_nhbins) { - for (szind_t i = 0; i < tcache_nhbins; i++) { - unsigned ncached_max = tcache_ncached_max_compute(i, - tcache_nhbins); +tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { + /* + * Compute the values for each bin, but for bins with indices larger + * than tcache_nbins, no items will be cached. + */ + for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) { + unsigned ncached_max = tcache_ncached_max_compute(i); + assert(ncached_max <= CACHE_BIN_NCACHED_MAX); cache_bin_info_init(&tcache_bin_info[i], ncached_max); } - for (szind_t i = tcache_nhbins; i < SC_NBINS; i++) { - /* Disabled small bins. */ - cache_bin_info_init(&tcache_bin_info[i], 0); - } } -/* Initialize auto tcache (embedded in TSD). */ -bool -tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) { +static bool +tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena, + cache_bin_info_t *tcache_bin_info) { tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd); tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); assert(cache_bin_still_zero_initialized(&tcache->bins[0])); - unsigned tcache_nhbins = tcache_nhbins_get(tcache); + unsigned tcache_nbins = tcache_nbins_get(tcache_slow); size_t size, alignment; - /* Takes 146B stack space. */ - cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {0}; - tcache_bin_info_compute(tcache_bin_info, tcache_nhbins); - cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins, + cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins, &size, &alignment); void *mem; @@ -860,6 +841,23 @@ tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) { return false; } +static bool +tsd_tcache_data_init_with_bin_settings(tsd_t *tsd, arena_t *arena, + cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { + assert(tcache_bin_info != NULL); + return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info); +} + +/* Initialize auto tcache (embedded in TSD). */ +static bool +tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) { + /* Takes 146B stack space. */ + cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}}; + tcache_bin_info_compute(tcache_bin_info); + + return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info); +} + /* Created manual tcache for tcache.create mallctl. */ tcache_t * tcache_create_explicit(tsd_t *tsd) { @@ -868,11 +866,11 @@ tcache_create_explicit(tsd_t *tsd) { * the beginning of the whole allocation (for freeing). The makes sure * the cache bins have the requested alignment. */ - unsigned tcache_nhbins = global_do_not_change_nhbins; + unsigned tcache_nbins = global_do_not_change_nbins; size_t tcache_size, alignment; - cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {0}; - tcache_bin_info_compute(tcache_bin_info, tcache_nhbins); - cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins, + cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}}; + tcache_bin_info_compute(tcache_bin_info); + cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins, &tcache_size, &alignment); size_t size = tcache_size + sizeof(tcache_t) @@ -889,7 +887,7 @@ tcache_create_explicit(tsd_t *tsd) { tcache_t *tcache = (void *)((byte_t *)mem + tcache_size); tcache_slow_t *tcache_slow = (void *)((byte_t *)mem + tcache_size + sizeof(tcache_t)); - tcache_max_and_nhbins_init(tcache); + tcache_default_settings_init(tcache_slow); tcache_init(tsd, tcache_slow, tcache, mem, tcache_bin_info); tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache, @@ -898,13 +896,83 @@ tcache_create_explicit(tsd_t *tsd) { return tcache; } +bool +tsd_tcache_enabled_data_init(tsd_t *tsd) { + /* Called upon tsd initialization. */ + tsd_tcache_enabled_set(tsd, opt_tcache); + /* + * tcache is not available yet, but we need to set up its tcache_nbins + * in advance. + */ + tcache_default_settings_init(tsd_tcache_slowp_get(tsd)); + tsd_slow_update(tsd); + + if (opt_tcache) { + /* Trigger tcache init. */ + tsd_tcache_data_init(tsd, NULL); + } + + return false; +} + +void +tcache_enabled_set(tsd_t *tsd, bool enabled) { + bool was_enabled = tsd_tcache_enabled_get(tsd); + + if (!was_enabled && enabled) { + tsd_tcache_data_init(tsd, NULL); + } else if (was_enabled && !enabled) { + tcache_cleanup(tsd); + } + /* Commit the state last. Above calls check current state. */ + tsd_tcache_enabled_set(tsd, enabled); + tsd_slow_update(tsd); +} + +void +thread_tcache_max_set(tsd_t *tsd, size_t tcache_max) { + assert(tcache_max <= TCACHE_MAXCLASS_LIMIT); + assert(tcache_max == sz_s2u(tcache_max)); + tcache_t *tcache = tsd_tcachep_get(tsd); + tcache_slow_t *tcache_slow = tcache->tcache_slow; + cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}}; + assert(tcache != NULL && tcache_slow != NULL); + + bool enabled = tcache_available(tsd); + arena_t *assigned_arena; + if (enabled) { + assigned_arena = tcache_slow->arena; + /* Carry over the bin settings during the reboot. */ + tcache_bin_settings_backup(tcache, tcache_bin_info); + /* Shutdown and reboot the tcache for a clean slate. */ + tcache_cleanup(tsd); + } + + /* + * Still set tcache_nbins of the tcache even if the tcache is not + * available yet because the values are stored in tsd_t and are + * always available for changing. + */ + tcache_max_set(tcache_slow, tcache_max); + + if (enabled) { + tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena, + tcache_bin_info); + } + + assert(tcache_nbins_get(tcache_slow) == sz_size2index(tcache_max) + 1); +} + static void tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { tcache_slow_t *tcache_slow = tcache->tcache_slow; assert(tcache_slow->arena != NULL); - for (unsigned i = 0; i < tcache_nhbins_get(tcache); i++) { + for (unsigned i = 0; i < tcache_nbins_get(tcache_slow); i++) { cache_bin_t *cache_bin = &tcache->bins[i]; + if (tcache_bin_disabled(i, cache_bin, tcache_slow)) { + continue; + } if (i < SC_NBINS) { tcache_bin_flush_small(tsd, tcache, cache_bin, i, 0); } else { @@ -974,8 +1042,7 @@ tcache_cleanup(tsd_t *tsd) { tcache_destroy(tsd, tcache, true); /* Make sure all bins used are reinitialized to the clean state. */ - memset(tcache->bins, 0, sizeof(cache_bin_t) * - tcache_nhbins_get(tcache)); + memset(tcache->bins, 0, sizeof(cache_bin_t) * TCACHE_NBINS_MAX); } void @@ -983,8 +1050,11 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { cassert(config_stats); /* Merge and reset tcache stats. */ - for (unsigned i = 0; i < tcache_nhbins_get(tcache); i++) { + for (unsigned i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) { cache_bin_t *cache_bin = &tcache->bins[i]; + if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) { + continue; + } if (i < SC_NBINS) { bin_t *bin = arena_bin_choose(tsdn, arena, i, NULL); malloc_mutex_lock(tsdn, &bin->lock); @@ -1110,7 +1180,7 @@ bool tcache_boot(tsdn_t *tsdn, base_t *base) { global_do_not_change_tcache_maxclass = sz_s2u(opt_tcache_max); assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT); - global_do_not_change_nhbins = + global_do_not_change_nbins = sz_size2index(global_do_not_change_tcache_maxclass) + 1; if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES, diff --git a/test/unit/cache_bin.c b/test/unit/cache_bin.c index 50d51a6d..aed34585 100644 --- a/test/unit/cache_bin.c +++ b/test/unit/cache_bin.c @@ -106,11 +106,13 @@ TEST_BEGIN(test_cache_bin) { cache_bin_info_init(&info, ncached_max); cache_bin_t bin; test_bin_init(&bin, &info); + cache_bin_info_t *bin_info = &bin.bin_info; /* Initialize to empty; should then have 0 elements. */ - expect_d_eq(ncached_max, cache_bin_info_ncached_max(&info), ""); - expect_true(cache_bin_ncached_get_local(&bin, &info) == 0, ""); - expect_true(cache_bin_low_water_get(&bin, &info) == 0, ""); + expect_d_eq(ncached_max, cache_bin_info_ncached_max_get(&bin, + &bin.bin_info), ""); + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, ""); + expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, ""); ptr = cache_bin_alloc_easy(&bin, &success); expect_false(success, "Shouldn't successfully allocate when empty"); @@ -127,14 +129,14 @@ TEST_BEGIN(test_cache_bin) { void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0); assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); for (cache_bin_sz_t i = 0; i < ncached_max; i++) { - expect_true(cache_bin_ncached_get_local(&bin, &info) == i, ""); + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == i, ""); success = cache_bin_dalloc_easy(&bin, &ptrs[i]); expect_true(success, "Should be able to dalloc into a non-full cache bin."); - expect_true(cache_bin_low_water_get(&bin, &info) == 0, + expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, "Pushes and pops shouldn't change low water of zero."); } - expect_true(cache_bin_ncached_get_local(&bin, &info) == ncached_max, + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max, ""); success = cache_bin_dalloc_easy(&bin, &ptrs[ncached_max]); expect_false(success, "Shouldn't be able to dalloc into a full bin."); @@ -142,9 +144,9 @@ TEST_BEGIN(test_cache_bin) { cache_bin_low_water_set(&bin); for (cache_bin_sz_t i = 0; i < ncached_max; i++) { - expect_true(cache_bin_low_water_get(&bin, &info) + expect_true(cache_bin_low_water_get(&bin, bin_info) == ncached_max - i, ""); - expect_true(cache_bin_ncached_get_local(&bin, &info) + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max - i, ""); /* * This should fail -- the easy variant can't change the low @@ -153,9 +155,9 @@ TEST_BEGIN(test_cache_bin) { ptr = cache_bin_alloc_easy(&bin, &success); expect_ptr_null(ptr, ""); expect_false(success, ""); - expect_true(cache_bin_low_water_get(&bin, &info) + expect_true(cache_bin_low_water_get(&bin, bin_info) == ncached_max - i, ""); - expect_true(cache_bin_ncached_get_local(&bin, &info) + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max - i, ""); /* This should succeed, though. */ @@ -163,13 +165,13 @@ TEST_BEGIN(test_cache_bin) { expect_true(success, ""); expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1], "Alloc should pop in stack order"); - expect_true(cache_bin_low_water_get(&bin, &info) + expect_true(cache_bin_low_water_get(&bin, bin_info) == ncached_max - i - 1, ""); - expect_true(cache_bin_ncached_get_local(&bin, &info) + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max - i - 1, ""); } /* Now we're empty -- all alloc attempts should fail. */ - expect_true(cache_bin_ncached_get_local(&bin, &info) == 0, ""); + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, ""); ptr = cache_bin_alloc_easy(&bin, &success); expect_ptr_null(ptr, ""); expect_false(success, ""); @@ -185,7 +187,7 @@ TEST_BEGIN(test_cache_bin) { for (cache_bin_sz_t i = ncached_max / 2; i < ncached_max; i++) { cache_bin_dalloc_easy(&bin, &ptrs[i]); } - expect_true(cache_bin_ncached_get_local(&bin, &info) == ncached_max, + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max, ""); for (cache_bin_sz_t i = ncached_max - 1; i >= ncached_max / 2; i--) { /* @@ -202,60 +204,64 @@ TEST_BEGIN(test_cache_bin) { expect_ptr_null(ptr, ""); /* We're going to test filling -- we must be empty to start. */ - while (cache_bin_ncached_get_local(&bin, &info)) { + while (cache_bin_ncached_get_local(&bin, bin_info)) { cache_bin_alloc(&bin, &success); expect_true(success, ""); } /* Test fill. */ /* Try to fill all, succeed fully. */ - do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max, ncached_max); + do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, + ncached_max); /* Try to fill all, succeed partially. */ - do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max, + do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, ncached_max / 2); /* Try to fill all, fail completely. */ - do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max, 0); + do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, 0); /* Try to fill some, succeed fully. */ - do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2, + do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, ncached_max / 2); /* Try to fill some, succeed partially. */ - do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2, + do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, ncached_max / 4); /* Try to fill some, fail completely. */ - do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2, 0); + do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, 0); - do_flush_test(&bin, &info, ptrs, ncached_max, ncached_max); - do_flush_test(&bin, &info, ptrs, ncached_max, ncached_max / 2); - do_flush_test(&bin, &info, ptrs, ncached_max, 0); - do_flush_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 2); - do_flush_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 4); - do_flush_test(&bin, &info, ptrs, ncached_max / 2, 0); + do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max); + do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2); + do_flush_test(&bin, bin_info, ptrs, ncached_max, 0); + do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 2); + do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 4); + do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, 0); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max * 2); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max / 2); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 2); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 1); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 0); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, ncached_max); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, + ncached_max * 2); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, ncached_max); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 2); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 1); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 0); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, + ncached_max / 2); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, + ncached_max); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 4); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 2); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 1); - do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 0); - do_batch_alloc_test(&bin, &info, ptrs, 2, ncached_max); - do_batch_alloc_test(&bin, &info, ptrs, 2, 2); - do_batch_alloc_test(&bin, &info, ptrs, 2, 1); - do_batch_alloc_test(&bin, &info, ptrs, 2, 0); - do_batch_alloc_test(&bin, &info, ptrs, 1, 2); - do_batch_alloc_test(&bin, &info, ptrs, 1, 1); - do_batch_alloc_test(&bin, &info, ptrs, 1, 0); - do_batch_alloc_test(&bin, &info, ptrs, 0, 2); - do_batch_alloc_test(&bin, &info, ptrs, 0, 1); - do_batch_alloc_test(&bin, &info, ptrs, 0, 0); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 2); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 1); + do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 0); + do_batch_alloc_test(&bin, bin_info, ptrs, 2, ncached_max); + do_batch_alloc_test(&bin, bin_info, ptrs, 2, 2); + do_batch_alloc_test(&bin, bin_info, ptrs, 2, 1); + do_batch_alloc_test(&bin, bin_info, ptrs, 2, 0); + do_batch_alloc_test(&bin, bin_info, ptrs, 1, 2); + do_batch_alloc_test(&bin, bin_info, ptrs, 1, 1); + do_batch_alloc_test(&bin, bin_info, ptrs, 1, 0); + do_batch_alloc_test(&bin, bin_info, ptrs, 0, 2); + do_batch_alloc_test(&bin, bin_info, ptrs, 0, 1); + do_batch_alloc_test(&bin, bin_info, ptrs, 0, 0); free(ptrs); } @@ -328,6 +334,7 @@ TEST_BEGIN(test_cache_bin_stash) { cache_bin_info_t info; cache_bin_info_init(&info, ncached_max); test_bin_init(&bin, &info); + cache_bin_info_t *bin_info = &bin.bin_info; /* * The content of this array is not accessed; instead the interior @@ -337,10 +344,10 @@ TEST_BEGIN(test_cache_bin_stash) { assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); bool ret; for (cache_bin_sz_t i = 0; i < ncached_max; i++) { - expect_true(cache_bin_ncached_get_local(&bin, &info) == + expect_true(cache_bin_ncached_get_local(&bin, bin_info) == (i / 2 + i % 2), "Wrong ncached value"); - expect_true(cache_bin_nstashed_get_local(&bin, &info) == i / 2, - "Wrong nstashed value"); + expect_true(cache_bin_nstashed_get_local(&bin, bin_info) == + i / 2, "Wrong nstashed value"); if (i % 2 == 0) { cache_bin_dalloc_easy(&bin, &ptrs[i]); } else { @@ -362,18 +369,23 @@ TEST_BEGIN(test_cache_bin_stash) { expect_true(diff % 2 == 0, "Should be able to alloc"); } else { expect_false(ret, "Should not alloc stashed"); - expect_true(cache_bin_nstashed_get_local(&bin, &info) == - ncached_max / 2, "Wrong nstashed value"); + expect_true(cache_bin_nstashed_get_local(&bin, + bin_info) == ncached_max / 2, + "Wrong nstashed value"); } } test_bin_init(&bin, &info); - do_flush_stashed_test(&bin, &info, ptrs, ncached_max, 0); - do_flush_stashed_test(&bin, &info, ptrs, 0, ncached_max); - do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 2); - do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 4, ncached_max / 2); - do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 4); - do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 4, ncached_max / 4); + do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max, 0); + do_flush_stashed_test(&bin, bin_info, ptrs, 0, ncached_max); + do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2, + ncached_max / 2); + do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4, + ncached_max / 2); + do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2, + ncached_max / 4); + do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4, + ncached_max / 4); } TEST_END diff --git a/test/unit/tcache_max.c b/test/unit/tcache_max.c index 6481504e..53752463 100644 --- a/test/unit/tcache_max.c +++ b/test/unit/tcache_max.c @@ -76,8 +76,11 @@ tcache_bytes_read_local(void) { size_t tcache_bytes = 0; tsd_t *tsd = tsd_fetch(); tcache_t *tcache = tcache_get(tsd); - for (szind_t i = 0; i < tcache_nhbins_get(tcache); i++) { + for (szind_t i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) { cache_bin_t *cache_bin = &tcache->bins[i]; + if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) { + continue; + } cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, &cache_bin->bin_info); tcache_bytes += ncached * sz_index2size(i); @@ -211,7 +214,7 @@ TEST_BEGIN(test_tcache_max) { TEST_END static size_t -tcache_max2nhbins(size_t tcache_max) { +tcache_max2nbins(size_t tcache_max) { return sz_size2index(tcache_max) + 1; } @@ -241,23 +244,24 @@ validate_tcache_stack(tcache_t *tcache) { static void * tcache_check(void *arg) { size_t old_tcache_max, new_tcache_max, min_tcache_max, sz; - unsigned tcache_nhbins; + unsigned tcache_nbins; tsd_t *tsd = tsd_fetch(); tcache_t *tcache = tsd_tcachep_get(tsd); + tcache_slow_t *tcache_slow = tcache->tcache_slow; sz = sizeof(size_t); new_tcache_max = *(size_t *)arg; min_tcache_max = 1; /* - * Check the default tcache_max and tcache_nhbins of each thread's + * Check the default tcache_max and tcache_nbins of each thread's * auto tcache. */ - old_tcache_max = tcache_max_get(tcache); + old_tcache_max = tcache_max_get(tcache_slow); expect_zu_eq(old_tcache_max, opt_tcache_max, "Unexpected default value for tcache_max"); - tcache_nhbins = tcache_nhbins_get(tcache); - expect_zu_eq(tcache_nhbins, (size_t)global_do_not_change_nhbins, - "Unexpected default value for tcache_nhbins"); + tcache_nbins = tcache_nbins_get(tcache_slow); + expect_zu_eq(tcache_nbins, (size_t)global_do_not_change_nbins, + "Unexpected default value for tcache_nbins"); validate_tcache_stack(tcache); /* @@ -275,12 +279,12 @@ tcache_check(void *arg) { assert_d_eq(mallctl("thread.tcache.max", NULL, NULL, (void *)&temp_tcache_max, sz),.0, "Unexpected.mallctl().failure"); - old_tcache_max = tcache_max_get(tcache); + old_tcache_max = tcache_max_get(tcache_slow); expect_zu_eq(old_tcache_max, TCACHE_MAXCLASS_LIMIT, "Unexpected value for tcache_max"); - tcache_nhbins = tcache_nhbins_get(tcache); - expect_zu_eq(tcache_nhbins, TCACHE_NBINS_MAX, - "Unexpected value for tcache_nhbins"); + tcache_nbins = tcache_nbins_get(tcache_slow); + expect_zu_eq(tcache_nbins, TCACHE_NBINS_MAX, + "Unexpected value for tcache_nbins"); assert_d_eq(mallctl("thread.tcache.max", (void *)&old_tcache_max, &sz, (void *)&min_tcache_max, sz),.0, @@ -294,10 +298,10 @@ tcache_check(void *arg) { (void *)&e0, bool_sz), 0, "Unexpected mallctl() error"); expect_false(e1, "Unexpected previous tcache state"); min_tcache_max = sz_s2u(min_tcache_max); - expect_zu_eq(tcache_max_get(tcache), min_tcache_max, + expect_zu_eq(tcache_max_get(tcache_slow), min_tcache_max, "Unexpected value for tcache_max"); - expect_zu_eq(tcache_nhbins_get(tcache), - tcache_max2nhbins(min_tcache_max), "Unexpected value for nhbins"); + expect_zu_eq(tcache_nbins_get(tcache_slow), + tcache_max2nbins(min_tcache_max), "Unexpected value for nbins"); assert_d_eq(mallctl("thread.tcache.max", (void *)&old_tcache_max, &sz, (void *)&new_tcache_max, sz),.0, @@ -307,18 +311,18 @@ tcache_check(void *arg) { validate_tcache_stack(tcache); /* - * Check the thread's tcache_max and nhbins both through mallctl + * Check the thread's tcache_max and nbins both through mallctl * and alloc tests. */ if (new_tcache_max > TCACHE_MAXCLASS_LIMIT) { new_tcache_max = TCACHE_MAXCLASS_LIMIT; } - old_tcache_max = tcache_max_get(tcache); + old_tcache_max = tcache_max_get(tcache_slow); expect_zu_eq(old_tcache_max, new_tcache_max, "Unexpected value for tcache_max"); - tcache_nhbins = tcache_nhbins_get(tcache); - expect_zu_eq(tcache_nhbins, tcache_max2nhbins(new_tcache_max), - "Unexpected value for tcache_nhbins"); + tcache_nbins = tcache_nbins_get(tcache_slow); + expect_zu_eq(tcache_nbins, tcache_max2nbins(new_tcache_max), + "Unexpected value for tcache_nbins"); for (unsigned alloc_option = alloc_option_start; alloc_option < alloc_option_end; alloc_option++) {