From e4817c8d89a2a413e835c4adeab5c5c4412f9235 Mon Sep 17 00:00:00 2001 From: Shirui Cheng Date: Tue, 24 Oct 2023 13:51:14 -0700 Subject: [PATCH] Cleanup cache_bin_info_t* info input args --- include/jemalloc/internal/arena_externs.h | 3 +- include/jemalloc/internal/cache_bin.h | 97 +++++----- include/jemalloc/internal/tcache_inlines.h | 7 +- src/arena.c | 13 +- src/cache_bin.c | 2 +- src/tcache.c | 56 +++--- test/unit/cache_bin.c | 200 ++++++++++----------- test/unit/tcache_max.c | 4 +- 8 files changed, 175 insertions(+), 207 deletions(-) diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index d79b607a..7a29fd8b 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -63,8 +63,7 @@ void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena); void arena_reset(tsd_t *tsd, arena_t *arena); void arena_destroy(tsd_t *tsd, arena_t *arena); void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, - cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind, - const unsigned nfill); + cache_bin_t *cache_bin, szind_t binind, const unsigned nfill); void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, bool slab); diff --git a/include/jemalloc/internal/cache_bin.h b/include/jemalloc/internal/cache_bin.h index 67565835..a26c3671 100644 --- a/include/jemalloc/internal/cache_bin.h +++ b/include/jemalloc/internal/cache_bin.h @@ -202,19 +202,19 @@ cache_bin_disabled(cache_bin_t *bin) { return disabled; } -/* Returns ncached_max: Upper limit on ncached. */ -static inline cache_bin_sz_t -cache_bin_info_ncached_max_get(cache_bin_t *bin, cache_bin_info_t *info) { - assert(!cache_bin_disabled(bin)); - assert(info == &bin->bin_info); - return info->ncached_max; -} - /* Gets ncached_max without asserting that the bin is enabled. */ static inline cache_bin_sz_t cache_bin_ncached_max_get_unsafe(cache_bin_t *bin) { return bin->bin_info.ncached_max; } + +/* Returns ncached_max: Upper limit on ncached. */ +static inline cache_bin_sz_t +cache_bin_ncached_max_get(cache_bin_t *bin) { + assert(!cache_bin_disabled(bin)); + return cache_bin_ncached_max_get_unsafe(bin); +} + /* * Internal. * @@ -267,9 +267,9 @@ cache_bin_ncached_get_internal(cache_bin_t *bin) { * possible. */ static inline cache_bin_sz_t -cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) { +cache_bin_ncached_get_local(cache_bin_t *bin) { cache_bin_sz_t n = cache_bin_ncached_get_internal(bin); - assert(n <= cache_bin_info_ncached_max_get(bin, info)); + assert(n <= cache_bin_ncached_max_get(bin)); return n; } @@ -304,9 +304,9 @@ cache_bin_empty_position_get(cache_bin_t *bin) { * arena statistics collection. */ static inline uint16_t -cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { +cache_bin_low_bits_low_bound_get(cache_bin_t *bin) { return (uint16_t)bin->low_bits_empty - - cache_bin_info_ncached_max_get(bin, info) * sizeof(void *); + cache_bin_ncached_max_get(bin) * sizeof(void *); } /* @@ -315,8 +315,8 @@ cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { * A pointer to the position with the lowest address of the backing array. */ static inline void ** -cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { - cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info); +cache_bin_low_bound_get(cache_bin_t *bin) { + cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin); void **ret = cache_bin_empty_position_get(bin) - ncached_max; assert(ret <= bin->stack_head); @@ -328,8 +328,8 @@ cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { * batch fill a nonempty cache bin. */ static inline void -cache_bin_assert_empty(cache_bin_t *bin, cache_bin_info_t *info) { - assert(cache_bin_ncached_get_local(bin, info) == 0); +cache_bin_assert_empty(cache_bin_t *bin) { + assert(cache_bin_ncached_get_local(bin) == 0); assert(cache_bin_empty_position_get(bin) == bin->stack_head); } @@ -346,10 +346,10 @@ cache_bin_low_water_get_internal(cache_bin_t *bin) { /* Returns the numeric value of low water in [0, ncached]. */ static inline cache_bin_sz_t -cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) { +cache_bin_low_water_get(cache_bin_t *bin) { cache_bin_sz_t low_water = cache_bin_low_water_get_internal(bin); - assert(low_water <= cache_bin_info_ncached_max_get(bin, info)); - assert(low_water <= cache_bin_ncached_get_local(bin, info)); + assert(low_water <= cache_bin_ncached_max_get(bin)); + assert(low_water <= cache_bin_ncached_get_local(bin)); cache_bin_assert_earlier(bin, (uint16_t)(uintptr_t)bin->stack_head, bin->low_bits_low_water); @@ -530,17 +530,16 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) { /* Get the number of stashed pointers. */ JEMALLOC_ALWAYS_INLINE cache_bin_sz_t -cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) { - cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info); - uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin, - info); +cache_bin_nstashed_get_internal(cache_bin_t *bin) { + cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin); + uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin); cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound, bin->low_bits_full) / sizeof(void *); assert(n <= ncached_max); if (config_debug && n != 0) { /* Below are for assertions only. */ - void **low_bound = cache_bin_low_bound_get(bin, info); + void **low_bound = cache_bin_low_bound_get(bin); assert((uint16_t)(uintptr_t)low_bound == low_bits_low_bound); void *stashed = *(low_bound + n - 1); @@ -556,9 +555,9 @@ cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) { } JEMALLOC_ALWAYS_INLINE cache_bin_sz_t -cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) { - cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin, info); - assert(n <= cache_bin_info_ncached_max_get(bin, info)); +cache_bin_nstashed_get_local(cache_bin_t *bin) { + cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin); + assert(n <= cache_bin_ncached_max_get(bin)); return n; } @@ -579,12 +578,12 @@ cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) { * This function should not call other utility functions because the racy * condition may cause unexpected / undefined behaviors in unverified utility * functions. Currently, this function calls two utility functions - * cache_bin_info_ncached_max_get and cache_bin_low_bits_low_bound_get because + * cache_bin_ncached_max_get and cache_bin_low_bits_low_bound_get because * they help access values that will not be concurrently modified. */ static inline void -cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_sz_t *ncached, cache_bin_sz_t *nstashed) { +cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_sz_t *ncached, + cache_bin_sz_t *nstashed) { /* Racy version of cache_bin_ncached_get_internal. */ cache_bin_sz_t diff = bin->low_bits_empty - (uint16_t)(uintptr_t)bin->stack_head; @@ -592,8 +591,7 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info, *ncached = n; /* Racy version of cache_bin_nstashed_get_internal. */ - uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin, - info); + uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin); n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *); *nstashed = n; /* @@ -643,9 +641,9 @@ struct cache_bin_ptr_array_s { * finish_fill call before doing any alloc/dalloc operations on the bin. */ static inline void -cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_ptr_array_t *arr, cache_bin_sz_t nfill) { - cache_bin_assert_empty(bin, info); +cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr, + cache_bin_sz_t nfill) { + cache_bin_assert_empty(bin); arr->ptr = cache_bin_empty_position_get(bin) - nfill; } @@ -655,9 +653,9 @@ cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info, * case of OOM. */ static inline void -cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_ptr_array_t *arr, cache_bin_sz_t nfilled) { - cache_bin_assert_empty(bin, info); +cache_bin_finish_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr, + cache_bin_sz_t nfilled) { + cache_bin_assert_empty(bin); void **empty_position = cache_bin_empty_position_get(bin); if (nfilled < arr->n) { memmove(empty_position - nfilled, empty_position - arr->n, @@ -671,17 +669,17 @@ cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info, * everything we give them. */ static inline void -cache_bin_init_ptr_array_for_flush(cache_bin_t *bin, cache_bin_info_t *info, +cache_bin_init_ptr_array_for_flush(cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) { arr->ptr = cache_bin_empty_position_get(bin) - nflush; - assert(cache_bin_ncached_get_local(bin, info) == 0 + assert(cache_bin_ncached_get_local(bin) == 0 || *arr->ptr != NULL); } static inline void -cache_bin_finish_flush(cache_bin_t *bin, cache_bin_info_t *info, - cache_bin_ptr_array_t *arr, cache_bin_sz_t nflushed) { - unsigned rem = cache_bin_ncached_get_local(bin, info) - nflushed; +cache_bin_finish_flush(cache_bin_t *bin, cache_bin_ptr_array_t *arr, + cache_bin_sz_t nflushed) { + unsigned rem = cache_bin_ncached_get_local(bin) - nflushed; memmove(bin->stack_head + nflushed, bin->stack_head, rem * sizeof(void *)); bin->stack_head += nflushed; @@ -690,23 +688,22 @@ cache_bin_finish_flush(cache_bin_t *bin, cache_bin_info_t *info, static inline void cache_bin_init_ptr_array_for_stashed(cache_bin_t *bin, szind_t binind, - cache_bin_info_t *info, cache_bin_ptr_array_t *arr, - cache_bin_sz_t nstashed) { + cache_bin_ptr_array_t *arr, cache_bin_sz_t nstashed) { assert(nstashed > 0); - assert(cache_bin_nstashed_get_local(bin, info) == nstashed); + assert(cache_bin_nstashed_get_local(bin) == nstashed); - void **low_bound = cache_bin_low_bound_get(bin, info); + void **low_bound = cache_bin_low_bound_get(bin); arr->ptr = low_bound; assert(*arr->ptr != NULL); } static inline void -cache_bin_finish_flush_stashed(cache_bin_t *bin, cache_bin_info_t *info) { - void **low_bound = cache_bin_low_bound_get(bin, info); +cache_bin_finish_flush_stashed(cache_bin_t *bin) { + void **low_bound = cache_bin_low_bound_get(bin); /* Reset the bin local full position. */ bin->low_bits_full = (uint16_t)(uintptr_t)low_bound; - assert(cache_bin_nstashed_get_local(bin, info) == 0); + assert(cache_bin_nstashed_get_local(bin) == 0); } /* diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index 05599a5b..e8e3b41f 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -200,8 +200,7 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, arena_dalloc_small(tsd_tsdn(tsd), ptr); return; } - cache_bin_sz_t max = cache_bin_info_ncached_max_get( - bin, &bin->bin_info); + cache_bin_sz_t max = cache_bin_ncached_max_get(bin); unsigned remain = max >> opt_lg_tcache_flush_small_div; tcache_bin_flush_small(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); @@ -221,8 +220,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin = &tcache->bins[binind]; if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { - unsigned remain = cache_bin_info_ncached_max_get( - bin, &bin->bin_info) >> opt_lg_tcache_flush_large_div; + unsigned remain = cache_bin_ncached_max_get(bin) >> + opt_lg_tcache_flush_large_div; tcache_bin_flush_large(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); diff --git a/src/arena.c b/src/arena.c index b4ead26a..4e923015 100644 --- a/src/arena.c +++ b/src/arena.c @@ -168,8 +168,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, } cache_bin_sz_t ncached, nstashed; - cache_bin_nitems_get_remote(cache_bin, - &cache_bin->bin_info, &ncached, &nstashed); + cache_bin_nitems_get_remote(cache_bin, &ncached, &nstashed); astats->tcache_bytes += ncached * sz_index2size(i); astats->tcache_stashed_bytes += nstashed * sz_index2size(i); @@ -1020,16 +1019,14 @@ arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, - cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind, - const unsigned nfill) { - assert(cache_bin_ncached_get_local(cache_bin, cache_bin_info) == 0); + cache_bin_t *cache_bin, szind_t binind, const unsigned nfill) { + assert(cache_bin_ncached_get_local(cache_bin) == 0); assert(nfill != 0); const bin_info_t *bin_info = &bin_infos[binind]; CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nfill); - cache_bin_init_ptr_array_for_fill(cache_bin, cache_bin_info, &ptrs, - nfill); + cache_bin_init_ptr_array_for_fill(cache_bin, &ptrs, nfill); /* * Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull * slabs. After both are exhausted, new slabs will be allocated through @@ -1143,7 +1140,7 @@ label_refill: fresh_slab = NULL; } - cache_bin_finish_fill(cache_bin, cache_bin_info, &ptrs, filled); + cache_bin_finish_fill(cache_bin, &ptrs, filled); arena_decay_tick(tsdn, arena); } diff --git a/src/cache_bin.c b/src/cache_bin.c index 0454dbde..6438705f 100644 --- a/src/cache_bin.c +++ b/src/cache_bin.c @@ -100,7 +100,7 @@ cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc, bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head); assert(free_spots == bin_stack_size); if (!cache_bin_disabled(bin)) { - assert(cache_bin_ncached_get_local(bin, &bin->bin_info) == 0); + assert(cache_bin_ncached_get_local(bin) == 0); } assert(cache_bin_empty_position_get(bin) == empty_position); diff --git a/src/tcache.c b/src/tcache.c index d57574ca..015bdb11 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -142,10 +142,8 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, cache_bin_t *cache_bin = &tcache->bins[szind]; assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow)); - cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, - &cache_bin->bin_info); - cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, - &cache_bin->bin_info); + cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); + cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin); assert(!tcache_slow->bin_refilled[szind]); size_t nflush = low_water - (low_water >> 2); @@ -168,8 +166,8 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, * Reduce fill count by 2X. Limit lg_fill_div such that * the fill count is always at least 1. */ - if ((cache_bin_info_ncached_max_get(cache_bin, &cache_bin->bin_info) - >> (tcache_slow->lg_fill_div[szind] + 1)) >= 1) { + if ((cache_bin_ncached_max_get(cache_bin) >> + (tcache_slow->lg_fill_div[szind] + 1)) >= 1) { tcache_slow->lg_fill_div[szind]++; } } @@ -181,10 +179,8 @@ tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, assert(szind >= SC_NBINS); cache_bin_t *cache_bin = &tcache->bins[szind]; assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow)); - cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, - &cache_bin->bin_info); - cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, - &cache_bin->bin_info); + cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); + cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin); tcache_bin_flush_large(tsd, tcache, cache_bin, szind, (unsigned)(ncached - low_water + (low_water >> 2))); } @@ -206,8 +202,7 @@ tcache_event(tsd_t *tsd) { } tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small); - cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, - &cache_bin->bin_info); + cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin); if (low_water > 0) { if (is_small) { tcache_gc_small(tsd, tcache_slow, tcache, szind); @@ -255,13 +250,12 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, assert(tcache_slow->arena != NULL); assert(!tcache_bin_disabled(binind, cache_bin, tcache_slow)); - unsigned nfill = cache_bin_info_ncached_max_get(cache_bin, - &cache_bin->bin_info) >> tcache_slow->lg_fill_div[binind]; + unsigned nfill = cache_bin_ncached_max_get(cache_bin) + >> tcache_slow->lg_fill_div[binind]; if (nfill == 0) { nfill = 1; } - arena_cache_bin_fill_small(tsdn, arena, cache_bin, - &cache_bin->bin_info, binind, nfill); + arena_cache_bin_fill_small(tsdn, arena, cache_bin, binind, nfill); tcache_slow->bin_refilled[binind] = true; ret = cache_bin_alloc(cache_bin, tcache_success); @@ -533,20 +527,17 @@ tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small); - cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, - &cache_bin->bin_info); + cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); assert((cache_bin_sz_t)rem <= ncached); unsigned nflush = ncached - rem; CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nflush); - cache_bin_init_ptr_array_for_flush(cache_bin, &cache_bin->bin_info, - &ptrs, nflush); + cache_bin_init_ptr_array_for_flush(cache_bin, &ptrs, nflush); tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nflush, small); - cache_bin_finish_flush(cache_bin, &cache_bin->bin_info, &ptrs, - ncached - rem); + cache_bin_finish_flush(cache_bin, &ptrs, ncached - rem); } void @@ -575,33 +566,30 @@ void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, bool is_small) { assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); - cache_bin_info_t *info = &cache_bin->bin_info; /* * The two below are for assertion only. The content of original cached * items remain unchanged -- the stashed items reside on the other end * of the stack. Checking the stack head and ncached to verify. */ void *head_content = *cache_bin->stack_head; - cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin, - info); + cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin); - cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin, info); - assert(orig_cached + nstashed <= - cache_bin_info_ncached_max_get(cache_bin, info)); + cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin); + assert(orig_cached + nstashed <= cache_bin_ncached_max_get(cache_bin)); if (nstashed == 0) { return; } CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nstashed); - cache_bin_init_ptr_array_for_stashed(cache_bin, binind, info, &ptrs, + cache_bin_init_ptr_array_for_stashed(cache_bin, binind, &ptrs, nstashed); san_check_stashed_ptrs(ptrs.ptr, nstashed, sz_index2size(binind)); tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nstashed, is_small); - cache_bin_finish_flush_stashed(cache_bin, info); + cache_bin_finish_flush_stashed(cache_bin); - assert(cache_bin_nstashed_get_local(cache_bin, info) == 0); - assert(cache_bin_ncached_get_local(cache_bin, info) == orig_cached); + assert(cache_bin_nstashed_get_local(cache_bin) == 0); + assert(cache_bin_ncached_get_local(cache_bin) == orig_cached); assert(head_content == *cache_bin->stack_head); } @@ -633,7 +621,7 @@ tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size, cache_bin_t *bin = &tcache->bins[bin_ind]; *ncached_max = tcache_bin_disabled(bin_ind, bin, tcache->tcache_slow) ? - 0: cache_bin_info_ncached_max_get(bin, &bin->bin_info); + 0: cache_bin_ncached_max_get(bin); return false; } @@ -1105,7 +1093,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { if (tsd_tcache) { cache_bin_t *cache_bin = &tcache->bins[0]; - cache_bin_assert_empty(cache_bin, &cache_bin->bin_info); + cache_bin_assert_empty(cache_bin); } if (tsd_tcache && cache_bin_stack_use_thp()) { b0_dalloc_tcache_stack(tsd_tsdn(tsd), tcache_slow->dyn_alloc); diff --git a/test/unit/cache_bin.c b/test/unit/cache_bin.c index aed34585..1bb750d7 100644 --- a/test/unit/cache_bin.c +++ b/test/unit/cache_bin.c @@ -1,19 +1,18 @@ #include "test/jemalloc_test.h" static void -do_fill_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, - cache_bin_sz_t ncached_max, cache_bin_sz_t nfill_attempt, - cache_bin_sz_t nfill_succeed) { +do_fill_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t ncached_max, + cache_bin_sz_t nfill_attempt, cache_bin_sz_t nfill_succeed) { bool success; void *ptr; - assert_true(cache_bin_ncached_get_local(bin, info) == 0, ""); + assert_true(cache_bin_ncached_get_local(bin) == 0, ""); CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill_attempt); - cache_bin_init_ptr_array_for_fill(bin, info, &arr, nfill_attempt); + cache_bin_init_ptr_array_for_fill(bin, &arr, nfill_attempt); for (cache_bin_sz_t i = 0; i < nfill_succeed; i++) { arr.ptr[i] = &ptrs[i]; } - cache_bin_finish_fill(bin, info, &arr, nfill_succeed); - expect_true(cache_bin_ncached_get_local(bin, info) == nfill_succeed, + cache_bin_finish_fill(bin, &arr, nfill_succeed); + expect_true(cache_bin_ncached_get_local(bin) == nfill_succeed, ""); cache_bin_low_water_set(bin); @@ -22,18 +21,18 @@ do_fill_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, expect_true(success, ""); expect_ptr_eq(ptr, (void *)&ptrs[i], "Should pop in order filled"); - expect_true(cache_bin_low_water_get(bin, info) + expect_true(cache_bin_low_water_get(bin) == nfill_succeed - i - 1, ""); } - expect_true(cache_bin_ncached_get_local(bin, info) == 0, ""); - expect_true(cache_bin_low_water_get(bin, info) == 0, ""); + expect_true(cache_bin_ncached_get_local(bin) == 0, ""); + expect_true(cache_bin_low_water_get(bin) == 0, ""); } static void -do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, - cache_bin_sz_t nfill, cache_bin_sz_t nflush) { +do_flush_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill, + cache_bin_sz_t nflush) { bool success; - assert_true(cache_bin_ncached_get_local(bin, info) == 0, ""); + assert_true(cache_bin_ncached_get_local(bin) == 0, ""); for (cache_bin_sz_t i = 0; i < nfill; i++) { success = cache_bin_dalloc_easy(bin, &ptrs[i]); @@ -41,30 +40,30 @@ do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, } CACHE_BIN_PTR_ARRAY_DECLARE(arr, nflush); - cache_bin_init_ptr_array_for_flush(bin, info, &arr, nflush); + cache_bin_init_ptr_array_for_flush(bin, &arr, nflush); for (cache_bin_sz_t i = 0; i < nflush; i++) { expect_ptr_eq(arr.ptr[i], &ptrs[nflush - i - 1], ""); } - cache_bin_finish_flush(bin, info, &arr, nflush); + cache_bin_finish_flush(bin, &arr, nflush); - expect_true(cache_bin_ncached_get_local(bin, info) == nfill - nflush, + expect_true(cache_bin_ncached_get_local(bin) == nfill - nflush, ""); - while (cache_bin_ncached_get_local(bin, info) > 0) { + while (cache_bin_ncached_get_local(bin) > 0) { cache_bin_alloc(bin, &success); } } static void -do_batch_alloc_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, - cache_bin_sz_t nfill, size_t batch) { - assert_true(cache_bin_ncached_get_local(bin, info) == 0, ""); +do_batch_alloc_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill, + size_t batch) { + assert_true(cache_bin_ncached_get_local(bin) == 0, ""); CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill); - cache_bin_init_ptr_array_for_fill(bin, info, &arr, nfill); + cache_bin_init_ptr_array_for_fill(bin, &arr, nfill); for (cache_bin_sz_t i = 0; i < nfill; i++) { arr.ptr[i] = &ptrs[i]; } - cache_bin_finish_fill(bin, info, &arr, nfill); - assert_true(cache_bin_ncached_get_local(bin, info) == nfill, ""); + cache_bin_finish_fill(bin, &arr, nfill); + assert_true(cache_bin_ncached_get_local(bin) == nfill, ""); cache_bin_low_water_set(bin); void **out = malloc((batch + 1) * sizeof(void *)); @@ -73,9 +72,9 @@ do_batch_alloc_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, for (cache_bin_sz_t i = 0; i < (cache_bin_sz_t)n; i++) { expect_ptr_eq(out[i], &ptrs[i], ""); } - expect_true(cache_bin_low_water_get(bin, info) == nfill - + expect_true(cache_bin_low_water_get(bin) == nfill - (cache_bin_sz_t)n, ""); - while (cache_bin_ncached_get_local(bin, info) > 0) { + while (cache_bin_ncached_get_local(bin) > 0) { bool success; cache_bin_alloc(bin, &success); } @@ -106,13 +105,11 @@ TEST_BEGIN(test_cache_bin) { cache_bin_info_init(&info, ncached_max); cache_bin_t bin; test_bin_init(&bin, &info); - cache_bin_info_t *bin_info = &bin.bin_info; /* Initialize to empty; should then have 0 elements. */ - expect_d_eq(ncached_max, cache_bin_info_ncached_max_get(&bin, - &bin.bin_info), ""); - expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, ""); - expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, ""); + expect_d_eq(ncached_max, cache_bin_ncached_max_get(&bin), ""); + expect_true(cache_bin_ncached_get_local(&bin) == 0, ""); + expect_true(cache_bin_low_water_get(&bin) == 0, ""); ptr = cache_bin_alloc_easy(&bin, &success); expect_false(success, "Shouldn't successfully allocate when empty"); @@ -129,14 +126,14 @@ TEST_BEGIN(test_cache_bin) { void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0); assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); for (cache_bin_sz_t i = 0; i < ncached_max; i++) { - expect_true(cache_bin_ncached_get_local(&bin, bin_info) == i, ""); + expect_true(cache_bin_ncached_get_local(&bin) == i, ""); success = cache_bin_dalloc_easy(&bin, &ptrs[i]); expect_true(success, "Should be able to dalloc into a non-full cache bin."); - expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, + expect_true(cache_bin_low_water_get(&bin) == 0, "Pushes and pops shouldn't change low water of zero."); } - expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max, + expect_true(cache_bin_ncached_get_local(&bin) == ncached_max, ""); success = cache_bin_dalloc_easy(&bin, &ptrs[ncached_max]); expect_false(success, "Shouldn't be able to dalloc into a full bin."); @@ -144,9 +141,9 @@ TEST_BEGIN(test_cache_bin) { cache_bin_low_water_set(&bin); for (cache_bin_sz_t i = 0; i < ncached_max; i++) { - expect_true(cache_bin_low_water_get(&bin, bin_info) + expect_true(cache_bin_low_water_get(&bin) == ncached_max - i, ""); - expect_true(cache_bin_ncached_get_local(&bin, bin_info) + expect_true(cache_bin_ncached_get_local(&bin) == ncached_max - i, ""); /* * This should fail -- the easy variant can't change the low @@ -155,9 +152,9 @@ TEST_BEGIN(test_cache_bin) { ptr = cache_bin_alloc_easy(&bin, &success); expect_ptr_null(ptr, ""); expect_false(success, ""); - expect_true(cache_bin_low_water_get(&bin, bin_info) + expect_true(cache_bin_low_water_get(&bin) == ncached_max - i, ""); - expect_true(cache_bin_ncached_get_local(&bin, bin_info) + expect_true(cache_bin_ncached_get_local(&bin) == ncached_max - i, ""); /* This should succeed, though. */ @@ -165,13 +162,13 @@ TEST_BEGIN(test_cache_bin) { expect_true(success, ""); expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1], "Alloc should pop in stack order"); - expect_true(cache_bin_low_water_get(&bin, bin_info) + expect_true(cache_bin_low_water_get(&bin) == ncached_max - i - 1, ""); - expect_true(cache_bin_ncached_get_local(&bin, bin_info) + expect_true(cache_bin_ncached_get_local(&bin) == ncached_max - i - 1, ""); } /* Now we're empty -- all alloc attempts should fail. */ - expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, ""); + expect_true(cache_bin_ncached_get_local(&bin) == 0, ""); ptr = cache_bin_alloc_easy(&bin, &success); expect_ptr_null(ptr, ""); expect_false(success, ""); @@ -187,7 +184,7 @@ TEST_BEGIN(test_cache_bin) { for (cache_bin_sz_t i = ncached_max / 2; i < ncached_max; i++) { cache_bin_dalloc_easy(&bin, &ptrs[i]); } - expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max, + expect_true(cache_bin_ncached_get_local(&bin) == ncached_max, ""); for (cache_bin_sz_t i = ncached_max - 1; i >= ncached_max / 2; i--) { /* @@ -204,77 +201,72 @@ TEST_BEGIN(test_cache_bin) { expect_ptr_null(ptr, ""); /* We're going to test filling -- we must be empty to start. */ - while (cache_bin_ncached_get_local(&bin, bin_info)) { + while (cache_bin_ncached_get_local(&bin)) { cache_bin_alloc(&bin, &success); expect_true(success, ""); } /* Test fill. */ /* Try to fill all, succeed fully. */ - do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, + do_fill_test(&bin, ptrs, ncached_max, ncached_max, ncached_max); /* Try to fill all, succeed partially. */ - do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, + do_fill_test(&bin, ptrs, ncached_max, ncached_max, ncached_max / 2); /* Try to fill all, fail completely. */ - do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, 0); + do_fill_test(&bin, ptrs, ncached_max, ncached_max, 0); /* Try to fill some, succeed fully. */ - do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, + do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2, ncached_max / 2); /* Try to fill some, succeed partially. */ - do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, + do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2, ncached_max / 4); /* Try to fill some, fail completely. */ - do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, 0); + do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2, 0); - do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max); - do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2); - do_flush_test(&bin, bin_info, ptrs, ncached_max, 0); - do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 2); - do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 4); - do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, 0); + do_flush_test(&bin, ptrs, ncached_max, ncached_max); + do_flush_test(&bin, ptrs, ncached_max, ncached_max / 2); + do_flush_test(&bin, ptrs, ncached_max, 0); + do_flush_test(&bin, ptrs, ncached_max / 2, ncached_max / 2); + do_flush_test(&bin, ptrs, ncached_max / 2, ncached_max / 4); + do_flush_test(&bin, ptrs, ncached_max / 2, 0); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, ncached_max); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, - ncached_max * 2); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, - ncached_max / 2); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 2); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 1); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 0); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, - ncached_max / 2); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, - ncached_max); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, - ncached_max / 4); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 2); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 1); - do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 0); - do_batch_alloc_test(&bin, bin_info, ptrs, 2, ncached_max); - do_batch_alloc_test(&bin, bin_info, ptrs, 2, 2); - do_batch_alloc_test(&bin, bin_info, ptrs, 2, 1); - do_batch_alloc_test(&bin, bin_info, ptrs, 2, 0); - do_batch_alloc_test(&bin, bin_info, ptrs, 1, 2); - do_batch_alloc_test(&bin, bin_info, ptrs, 1, 1); - do_batch_alloc_test(&bin, bin_info, ptrs, 1, 0); - do_batch_alloc_test(&bin, bin_info, ptrs, 0, 2); - do_batch_alloc_test(&bin, bin_info, ptrs, 0, 1); - do_batch_alloc_test(&bin, bin_info, ptrs, 0, 0); + do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max); + do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max * 2); + do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max / 2); + do_batch_alloc_test(&bin, ptrs, ncached_max, 2); + do_batch_alloc_test(&bin, ptrs, ncached_max, 1); + do_batch_alloc_test(&bin, ptrs, ncached_max, 0); + do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max / 2); + do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max); + do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max / 4); + do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 2); + do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 1); + do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 0); + do_batch_alloc_test(&bin, ptrs, 2, ncached_max); + do_batch_alloc_test(&bin, ptrs, 2, 2); + do_batch_alloc_test(&bin, ptrs, 2, 1); + do_batch_alloc_test(&bin, ptrs, 2, 0); + do_batch_alloc_test(&bin, ptrs, 1, 2); + do_batch_alloc_test(&bin, ptrs, 1, 1); + do_batch_alloc_test(&bin, ptrs, 1, 0); + do_batch_alloc_test(&bin, ptrs, 0, 2); + do_batch_alloc_test(&bin, ptrs, 0, 1); + do_batch_alloc_test(&bin, ptrs, 0, 0); free(ptrs); } TEST_END static void -do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, - cache_bin_sz_t nfill, cache_bin_sz_t nstash) { - expect_true(cache_bin_ncached_get_local(bin, info) == 0, +do_flush_stashed_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill, + cache_bin_sz_t nstash) { + expect_true(cache_bin_ncached_get_local(bin) == 0, "Bin not empty"); - expect_true(cache_bin_nstashed_get_local(bin, info) == 0, + expect_true(cache_bin_nstashed_get_local(bin) == 0, "Bin not empty"); - expect_true(nfill + nstash <= info->ncached_max, "Exceeded max"); + expect_true(nfill + nstash <= bin->bin_info.ncached_max, "Exceeded max"); bool ret; /* Fill */ @@ -282,7 +274,7 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, ret = cache_bin_dalloc_easy(bin, &ptrs[i]); expect_true(ret, "Unexpected fill failure"); } - expect_true(cache_bin_ncached_get_local(bin, info) == nfill, + expect_true(cache_bin_ncached_get_local(bin) == nfill, "Wrong cached count"); /* Stash */ @@ -290,10 +282,10 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, ret = cache_bin_stash(bin, &ptrs[i + nfill]); expect_true(ret, "Unexpected stash failure"); } - expect_true(cache_bin_nstashed_get_local(bin, info) == nstash, + expect_true(cache_bin_nstashed_get_local(bin) == nstash, "Wrong stashed count"); - if (nfill + nstash == info->ncached_max) { + if (nfill + nstash == bin->bin_info.ncached_max) { ret = cache_bin_dalloc_easy(bin, &ptrs[0]); expect_false(ret, "Should not dalloc into a full bin"); ret = cache_bin_stash(bin, &ptrs[0]); @@ -308,19 +300,19 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, expect_true((uintptr_t)ptr < (uintptr_t)&ptrs[nfill], "Should not alloc stashed ptrs"); } - expect_true(cache_bin_ncached_get_local(bin, info) == 0, + expect_true(cache_bin_ncached_get_local(bin) == 0, "Wrong cached count"); - expect_true(cache_bin_nstashed_get_local(bin, info) == nstash, + expect_true(cache_bin_nstashed_get_local(bin) == nstash, "Wrong stashed count"); cache_bin_alloc(bin, &ret); expect_false(ret, "Should not alloc stashed"); /* Clear stashed ones */ - cache_bin_finish_flush_stashed(bin, info); - expect_true(cache_bin_ncached_get_local(bin, info) == 0, + cache_bin_finish_flush_stashed(bin); + expect_true(cache_bin_ncached_get_local(bin) == 0, "Wrong cached count"); - expect_true(cache_bin_nstashed_get_local(bin, info) == 0, + expect_true(cache_bin_nstashed_get_local(bin) == 0, "Wrong stashed count"); cache_bin_alloc(bin, &ret); @@ -334,7 +326,6 @@ TEST_BEGIN(test_cache_bin_stash) { cache_bin_info_t info; cache_bin_info_init(&info, ncached_max); test_bin_init(&bin, &info); - cache_bin_info_t *bin_info = &bin.bin_info; /* * The content of this array is not accessed; instead the interior @@ -344,9 +335,9 @@ TEST_BEGIN(test_cache_bin_stash) { assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); bool ret; for (cache_bin_sz_t i = 0; i < ncached_max; i++) { - expect_true(cache_bin_ncached_get_local(&bin, bin_info) == + expect_true(cache_bin_ncached_get_local(&bin) == (i / 2 + i % 2), "Wrong ncached value"); - expect_true(cache_bin_nstashed_get_local(&bin, bin_info) == + expect_true(cache_bin_nstashed_get_local(&bin) == i / 2, "Wrong nstashed value"); if (i % 2 == 0) { cache_bin_dalloc_easy(&bin, &ptrs[i]); @@ -369,22 +360,21 @@ TEST_BEGIN(test_cache_bin_stash) { expect_true(diff % 2 == 0, "Should be able to alloc"); } else { expect_false(ret, "Should not alloc stashed"); - expect_true(cache_bin_nstashed_get_local(&bin, - bin_info) == ncached_max / 2, + expect_true(cache_bin_nstashed_get_local(&bin) == ncached_max / 2, "Wrong nstashed value"); } } test_bin_init(&bin, &info); - do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max, 0); - do_flush_stashed_test(&bin, bin_info, ptrs, 0, ncached_max); - do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2, + do_flush_stashed_test(&bin, ptrs, ncached_max, 0); + do_flush_stashed_test(&bin, ptrs, 0, ncached_max); + do_flush_stashed_test(&bin, ptrs, ncached_max / 2, ncached_max / 2); - do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4, + do_flush_stashed_test(&bin, ptrs, ncached_max / 4, ncached_max / 2); - do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2, + do_flush_stashed_test(&bin, ptrs, ncached_max / 2, ncached_max / 4); - do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4, + do_flush_stashed_test(&bin, ptrs, ncached_max / 4, ncached_max / 4); } TEST_END diff --git a/test/unit/tcache_max.c b/test/unit/tcache_max.c index c740b5e7..a64fca71 100644 --- a/test/unit/tcache_max.c +++ b/test/unit/tcache_max.c @@ -81,8 +81,7 @@ tcache_bytes_read_local(void) { if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) { continue; } - cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, - &cache_bin->bin_info); + cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); tcache_bytes += ncached * sz_index2size(i); } return tcache_bytes; @@ -370,4 +369,3 @@ main(void) { test_tcache_max, test_thread_tcache_max); } -