project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleanup cache_bin_info_t* info input args

Browse Source
This commit is contained in:
Shirui Cheng 2023-10-24 13:51:14 -07:00 committed by Qi Wang
parent 3025b021b9
commit e4817c8d89
8 changed files with 175 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/jemalloc/internal/arena_externs.h
View File

@ -63,8 +63,7 @@ void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
void arena_reset(tsd_t *tsd, arena_t *arena); void arena_reset(tsd_t *tsd, arena_t *arena);
void arena_destroy(tsd_t *tsd, arena_t *arena); void arena_destroy(tsd_t *tsd, arena_t *arena);
void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena,
cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind, cache_bin_t *cache_bin, szind_t binind, const unsigned nfill);
const unsigned nfill);
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
szind_t ind, bool zero, bool slab); szind_t ind, bool zero, bool slab);

97
include/jemalloc/internal/cache_bin.h
View File

@ -202,19 +202,19 @@ cache_bin_disabled(cache_bin_t *bin) {
return disabled; return disabled;
} }
/* Returns ncached_max: Upper limit on ncached. */
static inline cache_bin_sz_t
cache_bin_info_ncached_max_get(cache_bin_t *bin, cache_bin_info_t *info) {
assert(!cache_bin_disabled(bin));
assert(info == &bin->bin_info);
return info->ncached_max;
}
/* Gets ncached_max without asserting that the bin is enabled. */ /* Gets ncached_max without asserting that the bin is enabled. */
static inline cache_bin_sz_t static inline cache_bin_sz_t
cache_bin_ncached_max_get_unsafe(cache_bin_t *bin) { cache_bin_ncached_max_get_unsafe(cache_bin_t *bin) {
return bin->bin_info.ncached_max; return bin->bin_info.ncached_max;
} }
/* Returns ncached_max: Upper limit on ncached. */
static inline cache_bin_sz_t
cache_bin_ncached_max_get(cache_bin_t *bin) {
assert(!cache_bin_disabled(bin));
return cache_bin_ncached_max_get_unsafe(bin);
}
/* /*
* Internal. * Internal.
* *
@ -267,9 +267,9 @@ cache_bin_ncached_get_internal(cache_bin_t *bin) {
* possible. * possible.
*/ */
static inline cache_bin_sz_t static inline cache_bin_sz_t
cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_ncached_get_local(cache_bin_t *bin) {
cache_bin_sz_t n = cache_bin_ncached_get_internal(bin); cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
assert(n <= cache_bin_info_ncached_max_get(bin, info)); assert(n <= cache_bin_ncached_max_get(bin));
return n; return n;
} }
@ -304,9 +304,9 @@ cache_bin_empty_position_get(cache_bin_t *bin) {
* arena statistics collection. * arena statistics collection.
*/ */
static inline uint16_t static inline uint16_t
cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_low_bits_low_bound_get(cache_bin_t *bin) {
return (uint16_t)bin->low_bits_empty - return (uint16_t)bin->low_bits_empty -
cache_bin_info_ncached_max_get(bin, info) * sizeof(void *); cache_bin_ncached_max_get(bin) * sizeof(void *);
} }
/* /*
@ -315,8 +315,8 @@ cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
* A pointer to the position with the lowest address of the backing array. * A pointer to the position with the lowest address of the backing array.
*/ */
static inline void ** static inline void **
cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_low_bound_get(cache_bin_t *bin) {
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info); cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
void **ret = cache_bin_empty_position_get(bin) - ncached_max; void **ret = cache_bin_empty_position_get(bin) - ncached_max;
assert(ret <= bin->stack_head); assert(ret <= bin->stack_head);
@ -328,8 +328,8 @@ cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
* batch fill a nonempty cache bin. * batch fill a nonempty cache bin.
*/ */
static inline void static inline void
cache_bin_assert_empty(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_assert_empty(cache_bin_t *bin) {
assert(cache_bin_ncached_get_local(bin, info) == 0); assert(cache_bin_ncached_get_local(bin) == 0);
assert(cache_bin_empty_position_get(bin) == bin->stack_head); assert(cache_bin_empty_position_get(bin) == bin->stack_head);
} }
@ -346,10 +346,10 @@ cache_bin_low_water_get_internal(cache_bin_t *bin) {
/* Returns the numeric value of low water in [0, ncached]. */ /* Returns the numeric value of low water in [0, ncached]. */
static inline cache_bin_sz_t static inline cache_bin_sz_t
cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_low_water_get(cache_bin_t *bin) {
cache_bin_sz_t low_water = cache_bin_low_water_get_internal(bin); cache_bin_sz_t low_water = cache_bin_low_water_get_internal(bin);
assert(low_water <= cache_bin_info_ncached_max_get(bin, info)); assert(low_water <= cache_bin_ncached_max_get(bin));
assert(low_water <= cache_bin_ncached_get_local(bin, info)); assert(low_water <= cache_bin_ncached_get_local(bin));
cache_bin_assert_earlier(bin, (uint16_t)(uintptr_t)bin->stack_head, cache_bin_assert_earlier(bin, (uint16_t)(uintptr_t)bin->stack_head,
bin->low_bits_low_water); bin->low_bits_low_water);
@ -530,17 +530,16 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
/* Get the number of stashed pointers. */ /* Get the number of stashed pointers. */
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_nstashed_get_internal(cache_bin_t *bin) {
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info); cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin, uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin);
info);
cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound, cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound,
bin->low_bits_full) / sizeof(void *); bin->low_bits_full) / sizeof(void *);
assert(n <= ncached_max); assert(n <= ncached_max);
if (config_debug && n != 0) { if (config_debug && n != 0) {
/* Below are for assertions only. */ /* Below are for assertions only. */
void **low_bound = cache_bin_low_bound_get(bin, info); void **low_bound = cache_bin_low_bound_get(bin);
assert((uint16_t)(uintptr_t)low_bound == low_bits_low_bound); assert((uint16_t)(uintptr_t)low_bound == low_bits_low_bound);
void *stashed = *(low_bound + n - 1); void *stashed = *(low_bound + n - 1);
@ -556,9 +555,9 @@ cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) {
} }
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_nstashed_get_local(cache_bin_t *bin) {
cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin, info); cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin);
assert(n <= cache_bin_info_ncached_max_get(bin, info)); assert(n <= cache_bin_ncached_max_get(bin));
return n; return n;
} }
@ -579,12 +578,12 @@ cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
* This function should not call other utility functions because the racy * This function should not call other utility functions because the racy
* condition may cause unexpected / undefined behaviors in unverified utility * condition may cause unexpected / undefined behaviors in unverified utility
* functions. Currently, this function calls two utility functions * functions. Currently, this function calls two utility functions
* cache_bin_info_ncached_max_get and cache_bin_low_bits_low_bound_get because * cache_bin_ncached_max_get and cache_bin_low_bits_low_bound_get because
* they help access values that will not be concurrently modified. * they help access values that will not be concurrently modified.
*/ */
static inline void static inline void
cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info, cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_sz_t *ncached,
cache_bin_sz_t *ncached, cache_bin_sz_t *nstashed) { cache_bin_sz_t *nstashed) {
/* Racy version of cache_bin_ncached_get_internal. */ /* Racy version of cache_bin_ncached_get_internal. */
cache_bin_sz_t diff = bin->low_bits_empty - cache_bin_sz_t diff = bin->low_bits_empty -
(uint16_t)(uintptr_t)bin->stack_head; (uint16_t)(uintptr_t)bin->stack_head;
@ -592,8 +591,7 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info,
*ncached = n; *ncached = n;
/* Racy version of cache_bin_nstashed_get_internal. */ /* Racy version of cache_bin_nstashed_get_internal. */
uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin, uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin);
info);
n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *); n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *);
*nstashed = n; *nstashed = n;
/* /*
@ -643,9 +641,9 @@ struct cache_bin_ptr_array_s {
* finish_fill call before doing any alloc/dalloc operations on the bin. * finish_fill call before doing any alloc/dalloc operations on the bin.
*/ */
static inline void static inline void
cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info, cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr,
cache_bin_ptr_array_t *arr, cache_bin_sz_t nfill) { cache_bin_sz_t nfill) {
cache_bin_assert_empty(bin, info); cache_bin_assert_empty(bin);
arr->ptr = cache_bin_empty_position_get(bin) - nfill; arr->ptr = cache_bin_empty_position_get(bin) - nfill;
} }
@ -655,9 +653,9 @@ cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info,
* case of OOM. * case of OOM.
*/ */
static inline void static inline void
cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info, cache_bin_finish_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr,
cache_bin_ptr_array_t *arr, cache_bin_sz_t nfilled) { cache_bin_sz_t nfilled) {
cache_bin_assert_empty(bin, info); cache_bin_assert_empty(bin);
void **empty_position = cache_bin_empty_position_get(bin); void **empty_position = cache_bin_empty_position_get(bin);
if (nfilled < arr->n) { if (nfilled < arr->n) {
memmove(empty_position - nfilled, empty_position - arr->n, memmove(empty_position - nfilled, empty_position - arr->n,
@ -671,17 +669,17 @@ cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info,
* everything we give them. * everything we give them.
*/ */
static inline void static inline void
cache_bin_init_ptr_array_for_flush(cache_bin_t *bin, cache_bin_info_t *info, cache_bin_init_ptr_array_for_flush(cache_bin_t *bin,
cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) { cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) {
arr->ptr = cache_bin_empty_position_get(bin) - nflush; arr->ptr = cache_bin_empty_position_get(bin) - nflush;
assert(cache_bin_ncached_get_local(bin, info) == 0 assert(cache_bin_ncached_get_local(bin) == 0
|| *arr->ptr != NULL); || *arr->ptr != NULL);
} }
static inline void static inline void
cache_bin_finish_flush(cache_bin_t *bin, cache_bin_info_t *info, cache_bin_finish_flush(cache_bin_t *bin, cache_bin_ptr_array_t *arr,
cache_bin_ptr_array_t *arr, cache_bin_sz_t nflushed) { cache_bin_sz_t nflushed) {
unsigned rem = cache_bin_ncached_get_local(bin, info) - nflushed; unsigned rem = cache_bin_ncached_get_local(bin) - nflushed;
memmove(bin->stack_head + nflushed, bin->stack_head, memmove(bin->stack_head + nflushed, bin->stack_head,
rem * sizeof(void *)); rem * sizeof(void *));
bin->stack_head += nflushed; bin->stack_head += nflushed;
@ -690,23 +688,22 @@ cache_bin_finish_flush(cache_bin_t *bin, cache_bin_info_t *info,
static inline void static inline void
cache_bin_init_ptr_array_for_stashed(cache_bin_t *bin, szind_t binind, cache_bin_init_ptr_array_for_stashed(cache_bin_t *bin, szind_t binind,
cache_bin_info_t *info, cache_bin_ptr_array_t *arr, cache_bin_ptr_array_t *arr, cache_bin_sz_t nstashed) {
cache_bin_sz_t nstashed) {
assert(nstashed > 0); assert(nstashed > 0);
assert(cache_bin_nstashed_get_local(bin, info) == nstashed); assert(cache_bin_nstashed_get_local(bin) == nstashed);
void **low_bound = cache_bin_low_bound_get(bin, info); void **low_bound = cache_bin_low_bound_get(bin);
arr->ptr = low_bound; arr->ptr = low_bound;
assert(*arr->ptr != NULL); assert(*arr->ptr != NULL);
} }
static inline void static inline void
cache_bin_finish_flush_stashed(cache_bin_t *bin, cache_bin_info_t *info) { cache_bin_finish_flush_stashed(cache_bin_t *bin) {
void **low_bound = cache_bin_low_bound_get(bin, info); void **low_bound = cache_bin_low_bound_get(bin);
/* Reset the bin local full position. */ /* Reset the bin local full position. */
bin->low_bits_full = (uint16_t)(uintptr_t)low_bound; bin->low_bits_full = (uint16_t)(uintptr_t)low_bound;
assert(cache_bin_nstashed_get_local(bin, info) == 0); assert(cache_bin_nstashed_get_local(bin) == 0);
} }
/* /*

7
include/jemalloc/internal/tcache_inlines.h
View File

@ -200,8 +200,7 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
arena_dalloc_small(tsd_tsdn(tsd), ptr); arena_dalloc_small(tsd_tsdn(tsd), ptr);
return; return;
} }
cache_bin_sz_t max = cache_bin_info_ncached_max_get( cache_bin_sz_t max = cache_bin_ncached_max_get(bin);
bin, &bin->bin_info);
unsigned remain = max >> opt_lg_tcache_flush_small_div; unsigned remain = max >> opt_lg_tcache_flush_small_div;
tcache_bin_flush_small(tsd, tcache, bin, binind, remain); tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr); bool ret = cache_bin_dalloc_easy(bin, ptr);
@ -221,8 +220,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
cache_bin_t *bin = &tcache->bins[binind]; cache_bin_t *bin = &tcache->bins[binind];
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
unsigned remain = cache_bin_info_ncached_max_get( unsigned remain = cache_bin_ncached_max_get(bin) >>
bin, &bin->bin_info) >> opt_lg_tcache_flush_large_div; opt_lg_tcache_flush_large_div;
tcache_bin_flush_large(tsd, tcache, bin, binind, remain); tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr); bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret); assert(ret);

13
src/arena.c
View File

@ -168,8 +168,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
} }
cache_bin_sz_t ncached, nstashed; cache_bin_sz_t ncached, nstashed;
cache_bin_nitems_get_remote(cache_bin, cache_bin_nitems_get_remote(cache_bin, &ncached, &nstashed);
&cache_bin->bin_info, &ncached, &nstashed);
astats->tcache_bytes += ncached * sz_index2size(i); astats->tcache_bytes += ncached * sz_index2size(i);
astats->tcache_stashed_bytes += nstashed * astats->tcache_stashed_bytes += nstashed *
sz_index2size(i); sz_index2size(i);
@ -1020,16 +1019,14 @@ arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
void void
arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena,
cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind, cache_bin_t *cache_bin, szind_t binind, const unsigned nfill) {
const unsigned nfill) { assert(cache_bin_ncached_get_local(cache_bin) == 0);
assert(cache_bin_ncached_get_local(cache_bin, cache_bin_info) == 0);
assert(nfill != 0); assert(nfill != 0);
const bin_info_t *bin_info = &bin_infos[binind]; const bin_info_t *bin_info = &bin_infos[binind];
CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nfill); CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nfill);
cache_bin_init_ptr_array_for_fill(cache_bin, cache_bin_info, &ptrs, cache_bin_init_ptr_array_for_fill(cache_bin, &ptrs, nfill);
nfill);
/* /*
* Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull * Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull
* slabs. After both are exhausted, new slabs will be allocated through * slabs. After both are exhausted, new slabs will be allocated through
@ -1143,7 +1140,7 @@ label_refill:
fresh_slab = NULL; fresh_slab = NULL;
} }
cache_bin_finish_fill(cache_bin, cache_bin_info, &ptrs, filled); cache_bin_finish_fill(cache_bin, &ptrs, filled);
arena_decay_tick(tsdn, arena); arena_decay_tick(tsdn, arena);
} }

2
src/cache_bin.c
View File

@ -100,7 +100,7 @@ cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head); bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head);
assert(free_spots == bin_stack_size); assert(free_spots == bin_stack_size);
if (!cache_bin_disabled(bin)) { if (!cache_bin_disabled(bin)) {
assert(cache_bin_ncached_get_local(bin, &bin->bin_info) == 0); assert(cache_bin_ncached_get_local(bin) == 0);
} }
assert(cache_bin_empty_position_get(bin) == empty_position); assert(cache_bin_empty_position_get(bin) == empty_position);

56
src/tcache.c
View File

@ -142,10 +142,8 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
cache_bin_t *cache_bin = &tcache->bins[szind]; cache_bin_t *cache_bin = &tcache->bins[szind];
assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow)); assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow));
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin);
&cache_bin->bin_info); cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&cache_bin->bin_info);
assert(!tcache_slow->bin_refilled[szind]); assert(!tcache_slow->bin_refilled[szind]);
size_t nflush = low_water - (low_water >> 2); size_t nflush = low_water - (low_water >> 2);
@ -168,8 +166,8 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
* Reduce fill count by 2X. Limit lg_fill_div such that * Reduce fill count by 2X. Limit lg_fill_div such that
* the fill count is always at least 1. * the fill count is always at least 1.
*/ */
if ((cache_bin_info_ncached_max_get(cache_bin, &cache_bin->bin_info) if ((cache_bin_ncached_max_get(cache_bin) >>
>> (tcache_slow->lg_fill_div[szind] + 1)) >= 1) { (tcache_slow->lg_fill_div[szind] + 1)) >= 1) {
tcache_slow->lg_fill_div[szind]++; tcache_slow->lg_fill_div[szind]++;
} }
} }
@ -181,10 +179,8 @@ tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
assert(szind >= SC_NBINS); assert(szind >= SC_NBINS);
cache_bin_t *cache_bin = &tcache->bins[szind]; cache_bin_t *cache_bin = &tcache->bins[szind];
assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow)); assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow));
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin);
&cache_bin->bin_info); cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&cache_bin->bin_info);
tcache_bin_flush_large(tsd, tcache, cache_bin, szind, tcache_bin_flush_large(tsd, tcache, cache_bin, szind,
(unsigned)(ncached - low_water + (low_water >> 2))); (unsigned)(ncached - low_water + (low_water >> 2)));
} }
@ -206,8 +202,7 @@ tcache_event(tsd_t *tsd) {
} }
tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small); tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin, cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin);
&cache_bin->bin_info);
if (low_water > 0) { if (low_water > 0) {
if (is_small) { if (is_small) {
tcache_gc_small(tsd, tcache_slow, tcache, szind); tcache_gc_small(tsd, tcache_slow, tcache, szind);
@ -255,13 +250,12 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena,
assert(tcache_slow->arena != NULL); assert(tcache_slow->arena != NULL);
assert(!tcache_bin_disabled(binind, cache_bin, tcache_slow)); assert(!tcache_bin_disabled(binind, cache_bin, tcache_slow));
unsigned nfill = cache_bin_info_ncached_max_get(cache_bin, unsigned nfill = cache_bin_ncached_max_get(cache_bin)
&cache_bin->bin_info) >> tcache_slow->lg_fill_div[binind]; >> tcache_slow->lg_fill_div[binind];
if (nfill == 0) { if (nfill == 0) {
nfill = 1; nfill = 1;
} }
arena_cache_bin_fill_small(tsdn, arena, cache_bin, arena_cache_bin_fill_small(tsdn, arena, cache_bin, binind, nfill);
&cache_bin->bin_info, binind, nfill);
tcache_slow->bin_refilled[binind] = true; tcache_slow->bin_refilled[binind] = true;
ret = cache_bin_alloc(cache_bin, tcache_success); ret = cache_bin_alloc(cache_bin, tcache_success);
@ -533,20 +527,17 @@ tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow));
tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small); tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small);
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin);
&cache_bin->bin_info);
assert((cache_bin_sz_t)rem <= ncached); assert((cache_bin_sz_t)rem <= ncached);
unsigned nflush = ncached - rem; unsigned nflush = ncached - rem;
CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nflush); CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nflush);
cache_bin_init_ptr_array_for_flush(cache_bin, &cache_bin->bin_info, cache_bin_init_ptr_array_for_flush(cache_bin, &ptrs, nflush);
&ptrs, nflush);
tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nflush, tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nflush,
small); small);
cache_bin_finish_flush(cache_bin, &cache_bin->bin_info, &ptrs, cache_bin_finish_flush(cache_bin, &ptrs, ncached - rem);
ncached - rem);
} }
void void
@ -575,33 +566,30 @@ void
tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
szind_t binind, bool is_small) { szind_t binind, bool is_small) {
assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow));
cache_bin_info_t *info = &cache_bin->bin_info;
/* /*
* The two below are for assertion only. The content of original cached * The two below are for assertion only. The content of original cached
* items remain unchanged -- the stashed items reside on the other end * items remain unchanged -- the stashed items reside on the other end
* of the stack. Checking the stack head and ncached to verify. * of the stack. Checking the stack head and ncached to verify.
*/ */
void *head_content = *cache_bin->stack_head; void *head_content = *cache_bin->stack_head;
cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin, cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin);
info);
cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin, info); cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin);
assert(orig_cached + nstashed <= assert(orig_cached + nstashed <= cache_bin_ncached_max_get(cache_bin));
cache_bin_info_ncached_max_get(cache_bin, info));
if (nstashed == 0) { if (nstashed == 0) {
return; return;
} }
CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nstashed); CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nstashed);
cache_bin_init_ptr_array_for_stashed(cache_bin, binind, info, &ptrs, cache_bin_init_ptr_array_for_stashed(cache_bin, binind, &ptrs,
nstashed); nstashed);
san_check_stashed_ptrs(ptrs.ptr, nstashed, sz_index2size(binind)); san_check_stashed_ptrs(ptrs.ptr, nstashed, sz_index2size(binind));
tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nstashed, tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nstashed,
is_small); is_small);
cache_bin_finish_flush_stashed(cache_bin, info); cache_bin_finish_flush_stashed(cache_bin);
assert(cache_bin_nstashed_get_local(cache_bin, info) == 0); assert(cache_bin_nstashed_get_local(cache_bin) == 0);
assert(cache_bin_ncached_get_local(cache_bin, info) == orig_cached); assert(cache_bin_ncached_get_local(cache_bin) == orig_cached);
assert(head_content == *cache_bin->stack_head); assert(head_content == *cache_bin->stack_head);
} }
@ -633,7 +621,7 @@ tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size,
cache_bin_t *bin = &tcache->bins[bin_ind]; cache_bin_t *bin = &tcache->bins[bin_ind];
*ncached_max = tcache_bin_disabled(bin_ind, bin, tcache->tcache_slow) ? *ncached_max = tcache_bin_disabled(bin_ind, bin, tcache->tcache_slow) ?
0: cache_bin_info_ncached_max_get(bin, &bin->bin_info); 0: cache_bin_ncached_max_get(bin);
return false; return false;
} }
@ -1105,7 +1093,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
if (tsd_tcache) { if (tsd_tcache) {
cache_bin_t *cache_bin = &tcache->bins[0]; cache_bin_t *cache_bin = &tcache->bins[0];
cache_bin_assert_empty(cache_bin, &cache_bin->bin_info); cache_bin_assert_empty(cache_bin);
} }
if (tsd_tcache && cache_bin_stack_use_thp()) { if (tsd_tcache && cache_bin_stack_use_thp()) {
b0_dalloc_tcache_stack(tsd_tsdn(tsd), tcache_slow->dyn_alloc); b0_dalloc_tcache_stack(tsd_tsdn(tsd), tcache_slow->dyn_alloc);

200
test/unit/cache_bin.c
View File

@ -1,19 +1,18 @@
#include "test/jemalloc_test.h" #include "test/jemalloc_test.h"
static void static void
do_fill_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, do_fill_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t ncached_max,
cache_bin_sz_t ncached_max, cache_bin_sz_t nfill_attempt, cache_bin_sz_t nfill_attempt, cache_bin_sz_t nfill_succeed) {
cache_bin_sz_t nfill_succeed) {
bool success; bool success;
void *ptr; void *ptr;
assert_true(cache_bin_ncached_get_local(bin, info) == 0, ""); assert_true(cache_bin_ncached_get_local(bin) == 0, "");
CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill_attempt); CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill_attempt);
cache_bin_init_ptr_array_for_fill(bin, info, &arr, nfill_attempt); cache_bin_init_ptr_array_for_fill(bin, &arr, nfill_attempt);
for (cache_bin_sz_t i = 0; i < nfill_succeed; i++) { for (cache_bin_sz_t i = 0; i < nfill_succeed; i++) {
arr.ptr[i] = &ptrs[i]; arr.ptr[i] = &ptrs[i];
} }
cache_bin_finish_fill(bin, info, &arr, nfill_succeed); cache_bin_finish_fill(bin, &arr, nfill_succeed);
expect_true(cache_bin_ncached_get_local(bin, info) == nfill_succeed, expect_true(cache_bin_ncached_get_local(bin) == nfill_succeed,
""); "");
cache_bin_low_water_set(bin); cache_bin_low_water_set(bin);
@ -22,18 +21,18 @@ do_fill_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
expect_true(success, ""); expect_true(success, "");
expect_ptr_eq(ptr, (void *)&ptrs[i], expect_ptr_eq(ptr, (void *)&ptrs[i],
"Should pop in order filled"); "Should pop in order filled");
expect_true(cache_bin_low_water_get(bin, info) expect_true(cache_bin_low_water_get(bin)
== nfill_succeed - i - 1, ""); == nfill_succeed - i - 1, "");
} }
expect_true(cache_bin_ncached_get_local(bin, info) == 0, ""); expect_true(cache_bin_ncached_get_local(bin) == 0, "");
expect_true(cache_bin_low_water_get(bin, info) == 0, ""); expect_true(cache_bin_low_water_get(bin) == 0, "");
} }
static void static void
do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, do_flush_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
cache_bin_sz_t nfill, cache_bin_sz_t nflush) { cache_bin_sz_t nflush) {
bool success; bool success;
assert_true(cache_bin_ncached_get_local(bin, info) == 0, ""); assert_true(cache_bin_ncached_get_local(bin) == 0, "");
for (cache_bin_sz_t i = 0; i < nfill; i++) { for (cache_bin_sz_t i = 0; i < nfill; i++) {
success = cache_bin_dalloc_easy(bin, &ptrs[i]); success = cache_bin_dalloc_easy(bin, &ptrs[i]);
@ -41,30 +40,30 @@ do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
} }
CACHE_BIN_PTR_ARRAY_DECLARE(arr, nflush); CACHE_BIN_PTR_ARRAY_DECLARE(arr, nflush);
cache_bin_init_ptr_array_for_flush(bin, info, &arr, nflush); cache_bin_init_ptr_array_for_flush(bin, &arr, nflush);
for (cache_bin_sz_t i = 0; i < nflush; i++) { for (cache_bin_sz_t i = 0; i < nflush; i++) {
expect_ptr_eq(arr.ptr[i], &ptrs[nflush - i - 1], ""); expect_ptr_eq(arr.ptr[i], &ptrs[nflush - i - 1], "");
} }
cache_bin_finish_flush(bin, info, &arr, nflush); cache_bin_finish_flush(bin, &arr, nflush);
expect_true(cache_bin_ncached_get_local(bin, info) == nfill - nflush, expect_true(cache_bin_ncached_get_local(bin) == nfill - nflush,
""); "");
while (cache_bin_ncached_get_local(bin, info) > 0) { while (cache_bin_ncached_get_local(bin) > 0) {
cache_bin_alloc(bin, &success); cache_bin_alloc(bin, &success);
} }
} }
static void static void
do_batch_alloc_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, do_batch_alloc_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
cache_bin_sz_t nfill, size_t batch) { size_t batch) {
assert_true(cache_bin_ncached_get_local(bin, info) == 0, ""); assert_true(cache_bin_ncached_get_local(bin) == 0, "");
CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill); CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill);
cache_bin_init_ptr_array_for_fill(bin, info, &arr, nfill); cache_bin_init_ptr_array_for_fill(bin, &arr, nfill);
for (cache_bin_sz_t i = 0; i < nfill; i++) { for (cache_bin_sz_t i = 0; i < nfill; i++) {
arr.ptr[i] = &ptrs[i]; arr.ptr[i] = &ptrs[i];
} }
cache_bin_finish_fill(bin, info, &arr, nfill); cache_bin_finish_fill(bin, &arr, nfill);
assert_true(cache_bin_ncached_get_local(bin, info) == nfill, ""); assert_true(cache_bin_ncached_get_local(bin) == nfill, "");
cache_bin_low_water_set(bin); cache_bin_low_water_set(bin);
void **out = malloc((batch + 1) * sizeof(void *)); void **out = malloc((batch + 1) * sizeof(void *));
@ -73,9 +72,9 @@ do_batch_alloc_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
for (cache_bin_sz_t i = 0; i < (cache_bin_sz_t)n; i++) { for (cache_bin_sz_t i = 0; i < (cache_bin_sz_t)n; i++) {
expect_ptr_eq(out[i], &ptrs[i], ""); expect_ptr_eq(out[i], &ptrs[i], "");
} }
expect_true(cache_bin_low_water_get(bin, info) == nfill - expect_true(cache_bin_low_water_get(bin) == nfill -
(cache_bin_sz_t)n, ""); (cache_bin_sz_t)n, "");
while (cache_bin_ncached_get_local(bin, info) > 0) { while (cache_bin_ncached_get_local(bin) > 0) {
bool success; bool success;
cache_bin_alloc(bin, &success); cache_bin_alloc(bin, &success);
} }
@ -106,13 +105,11 @@ TEST_BEGIN(test_cache_bin) {
cache_bin_info_init(&info, ncached_max); cache_bin_info_init(&info, ncached_max);
cache_bin_t bin; cache_bin_t bin;
test_bin_init(&bin, &info); test_bin_init(&bin, &info);
cache_bin_info_t *bin_info = &bin.bin_info;
/* Initialize to empty; should then have 0 elements. */ /* Initialize to empty; should then have 0 elements. */
expect_d_eq(ncached_max, cache_bin_info_ncached_max_get(&bin, expect_d_eq(ncached_max, cache_bin_ncached_max_get(&bin), "");
&bin.bin_info), ""); expect_true(cache_bin_ncached_get_local(&bin) == 0, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, ""); expect_true(cache_bin_low_water_get(&bin) == 0, "");
expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, "");
ptr = cache_bin_alloc_easy(&bin, &success); ptr = cache_bin_alloc_easy(&bin, &success);
expect_false(success, "Shouldn't successfully allocate when empty"); expect_false(success, "Shouldn't successfully allocate when empty");
@ -129,14 +126,14 @@ TEST_BEGIN(test_cache_bin) {
void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0); void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0);
assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
for (cache_bin_sz_t i = 0; i < ncached_max; i++) { for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == i, ""); expect_true(cache_bin_ncached_get_local(&bin) == i, "");
success = cache_bin_dalloc_easy(&bin, &ptrs[i]); success = cache_bin_dalloc_easy(&bin, &ptrs[i]);
expect_true(success, expect_true(success,
"Should be able to dalloc into a non-full cache bin."); "Should be able to dalloc into a non-full cache bin.");
expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, expect_true(cache_bin_low_water_get(&bin) == 0,
"Pushes and pops shouldn't change low water of zero."); "Pushes and pops shouldn't change low water of zero.");
} }
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max, expect_true(cache_bin_ncached_get_local(&bin) == ncached_max,
""); "");
success = cache_bin_dalloc_easy(&bin, &ptrs[ncached_max]); success = cache_bin_dalloc_easy(&bin, &ptrs[ncached_max]);
expect_false(success, "Shouldn't be able to dalloc into a full bin."); expect_false(success, "Shouldn't be able to dalloc into a full bin.");
@ -144,9 +141,9 @@ TEST_BEGIN(test_cache_bin) {
cache_bin_low_water_set(&bin); cache_bin_low_water_set(&bin);
for (cache_bin_sz_t i = 0; i < ncached_max; i++) { for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_low_water_get(&bin, bin_info) expect_true(cache_bin_low_water_get(&bin)
== ncached_max - i, ""); == ncached_max - i, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) expect_true(cache_bin_ncached_get_local(&bin)
== ncached_max - i, ""); == ncached_max - i, "");
/* /*
* This should fail -- the easy variant can't change the low * This should fail -- the easy variant can't change the low
@ -155,9 +152,9 @@ TEST_BEGIN(test_cache_bin) {
ptr = cache_bin_alloc_easy(&bin, &success); ptr = cache_bin_alloc_easy(&bin, &success);
expect_ptr_null(ptr, ""); expect_ptr_null(ptr, "");
expect_false(success, ""); expect_false(success, "");
expect_true(cache_bin_low_water_get(&bin, bin_info) expect_true(cache_bin_low_water_get(&bin)
== ncached_max - i, ""); == ncached_max - i, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) expect_true(cache_bin_ncached_get_local(&bin)
== ncached_max - i, ""); == ncached_max - i, "");
/* This should succeed, though. */ /* This should succeed, though. */
@ -165,13 +162,13 @@ TEST_BEGIN(test_cache_bin) {
expect_true(success, ""); expect_true(success, "");
expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1], expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1],
"Alloc should pop in stack order"); "Alloc should pop in stack order");
expect_true(cache_bin_low_water_get(&bin, bin_info) expect_true(cache_bin_low_water_get(&bin)
== ncached_max - i - 1, ""); == ncached_max - i - 1, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) expect_true(cache_bin_ncached_get_local(&bin)
== ncached_max - i - 1, ""); == ncached_max - i - 1, "");
} }
/* Now we're empty -- all alloc attempts should fail. */ /* Now we're empty -- all alloc attempts should fail. */
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, ""); expect_true(cache_bin_ncached_get_local(&bin) == 0, "");
ptr = cache_bin_alloc_easy(&bin, &success); ptr = cache_bin_alloc_easy(&bin, &success);
expect_ptr_null(ptr, ""); expect_ptr_null(ptr, "");
expect_false(success, ""); expect_false(success, "");
@ -187,7 +184,7 @@ TEST_BEGIN(test_cache_bin) {
for (cache_bin_sz_t i = ncached_max / 2; i < ncached_max; i++) { for (cache_bin_sz_t i = ncached_max / 2; i < ncached_max; i++) {
cache_bin_dalloc_easy(&bin, &ptrs[i]); cache_bin_dalloc_easy(&bin, &ptrs[i]);
} }
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max, expect_true(cache_bin_ncached_get_local(&bin) == ncached_max,
""); "");
for (cache_bin_sz_t i = ncached_max - 1; i >= ncached_max / 2; i--) { for (cache_bin_sz_t i = ncached_max - 1; i >= ncached_max / 2; i--) {
/* /*
@ -204,77 +201,72 @@ TEST_BEGIN(test_cache_bin) {
expect_ptr_null(ptr, ""); expect_ptr_null(ptr, "");
/* We're going to test filling -- we must be empty to start. */ /* We're going to test filling -- we must be empty to start. */
while (cache_bin_ncached_get_local(&bin, bin_info)) { while (cache_bin_ncached_get_local(&bin)) {
cache_bin_alloc(&bin, &success); cache_bin_alloc(&bin, &success);
expect_true(success, ""); expect_true(success, "");
} }
/* Test fill. */ /* Test fill. */
/* Try to fill all, succeed fully. */ /* Try to fill all, succeed fully. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, do_fill_test(&bin, ptrs, ncached_max, ncached_max,
ncached_max); ncached_max);
/* Try to fill all, succeed partially. */ /* Try to fill all, succeed partially. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, do_fill_test(&bin, ptrs, ncached_max, ncached_max,
ncached_max / 2); ncached_max / 2);
/* Try to fill all, fail completely. */ /* Try to fill all, fail completely. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, 0); do_fill_test(&bin, ptrs, ncached_max, ncached_max, 0);
/* Try to fill some, succeed fully. */ /* Try to fill some, succeed fully. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2,
ncached_max / 2); ncached_max / 2);
/* Try to fill some, succeed partially. */ /* Try to fill some, succeed partially. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2,
ncached_max / 4); ncached_max / 4);
/* Try to fill some, fail completely. */ /* Try to fill some, fail completely. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, 0); do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2, 0);
do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max); do_flush_test(&bin, ptrs, ncached_max, ncached_max);
do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2); do_flush_test(&bin, ptrs, ncached_max, ncached_max / 2);
do_flush_test(&bin, bin_info, ptrs, ncached_max, 0); do_flush_test(&bin, ptrs, ncached_max, 0);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 2); do_flush_test(&bin, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 4); do_flush_test(&bin, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, 0); do_flush_test(&bin, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, ncached_max); do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max * 2);
ncached_max * 2); do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max / 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, do_batch_alloc_test(&bin, ptrs, ncached_max, 2);
ncached_max / 2); do_batch_alloc_test(&bin, ptrs, ncached_max, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 2); do_batch_alloc_test(&bin, ptrs, ncached_max, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 1); do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max / 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 0); do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max / 4);
ncached_max / 2); do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 1);
ncached_max); do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, do_batch_alloc_test(&bin, ptrs, 2, ncached_max);
ncached_max / 4); do_batch_alloc_test(&bin, ptrs, 2, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 2); do_batch_alloc_test(&bin, ptrs, 2, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 1); do_batch_alloc_test(&bin, ptrs, 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 0); do_batch_alloc_test(&bin, ptrs, 1, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, ncached_max); do_batch_alloc_test(&bin, ptrs, 1, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 2); do_batch_alloc_test(&bin, ptrs, 1, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 1); do_batch_alloc_test(&bin, ptrs, 0, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 0); do_batch_alloc_test(&bin, ptrs, 0, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 2); do_batch_alloc_test(&bin, ptrs, 0, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 0);
free(ptrs); free(ptrs);
} }
TEST_END TEST_END
static void static void
do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, do_flush_stashed_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
cache_bin_sz_t nfill, cache_bin_sz_t nstash) { cache_bin_sz_t nstash) {
expect_true(cache_bin_ncached_get_local(bin, info) == 0, expect_true(cache_bin_ncached_get_local(bin) == 0,
"Bin not empty"); "Bin not empty");
expect_true(cache_bin_nstashed_get_local(bin, info) == 0, expect_true(cache_bin_nstashed_get_local(bin) == 0,
"Bin not empty"); "Bin not empty");
expect_true(nfill + nstash <= info->ncached_max, "Exceeded max"); expect_true(nfill + nstash <= bin->bin_info.ncached_max, "Exceeded max");
bool ret; bool ret;
/* Fill */ /* Fill */
@ -282,7 +274,7 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
ret = cache_bin_dalloc_easy(bin, &ptrs[i]); ret = cache_bin_dalloc_easy(bin, &ptrs[i]);
expect_true(ret, "Unexpected fill failure"); expect_true(ret, "Unexpected fill failure");
} }
expect_true(cache_bin_ncached_get_local(bin, info) == nfill, expect_true(cache_bin_ncached_get_local(bin) == nfill,
"Wrong cached count"); "Wrong cached count");
/* Stash */ /* Stash */
@ -290,10 +282,10 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
ret = cache_bin_stash(bin, &ptrs[i + nfill]); ret = cache_bin_stash(bin, &ptrs[i + nfill]);
expect_true(ret, "Unexpected stash failure"); expect_true(ret, "Unexpected stash failure");
} }
expect_true(cache_bin_nstashed_get_local(bin, info) == nstash, expect_true(cache_bin_nstashed_get_local(bin) == nstash,
"Wrong stashed count"); "Wrong stashed count");
if (nfill + nstash == info->ncached_max) { if (nfill + nstash == bin->bin_info.ncached_max) {
ret = cache_bin_dalloc_easy(bin, &ptrs[0]); ret = cache_bin_dalloc_easy(bin, &ptrs[0]);
expect_false(ret, "Should not dalloc into a full bin"); expect_false(ret, "Should not dalloc into a full bin");
ret = cache_bin_stash(bin, &ptrs[0]); ret = cache_bin_stash(bin, &ptrs[0]);
@ -308,19 +300,19 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
expect_true((uintptr_t)ptr < (uintptr_t)&ptrs[nfill], expect_true((uintptr_t)ptr < (uintptr_t)&ptrs[nfill],
"Should not alloc stashed ptrs"); "Should not alloc stashed ptrs");
} }
expect_true(cache_bin_ncached_get_local(bin, info) == 0, expect_true(cache_bin_ncached_get_local(bin) == 0,
"Wrong cached count"); "Wrong cached count");
expect_true(cache_bin_nstashed_get_local(bin, info) == nstash, expect_true(cache_bin_nstashed_get_local(bin) == nstash,
"Wrong stashed count"); "Wrong stashed count");
cache_bin_alloc(bin, &ret); cache_bin_alloc(bin, &ret);
expect_false(ret, "Should not alloc stashed"); expect_false(ret, "Should not alloc stashed");
/* Clear stashed ones */ /* Clear stashed ones */
cache_bin_finish_flush_stashed(bin, info); cache_bin_finish_flush_stashed(bin);
expect_true(cache_bin_ncached_get_local(bin, info) == 0, expect_true(cache_bin_ncached_get_local(bin) == 0,
"Wrong cached count"); "Wrong cached count");
expect_true(cache_bin_nstashed_get_local(bin, info) == 0, expect_true(cache_bin_nstashed_get_local(bin) == 0,
"Wrong stashed count"); "Wrong stashed count");
cache_bin_alloc(bin, &ret); cache_bin_alloc(bin, &ret);
@ -334,7 +326,6 @@ TEST_BEGIN(test_cache_bin_stash) {
cache_bin_info_t info; cache_bin_info_t info;
cache_bin_info_init(&info, ncached_max); cache_bin_info_init(&info, ncached_max);
test_bin_init(&bin, &info); test_bin_init(&bin, &info);
cache_bin_info_t *bin_info = &bin.bin_info;
/* /*
* The content of this array is not accessed; instead the interior * The content of this array is not accessed; instead the interior
@ -344,9 +335,9 @@ TEST_BEGIN(test_cache_bin_stash) {
assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
bool ret; bool ret;
for (cache_bin_sz_t i = 0; i < ncached_max; i++) { for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == expect_true(cache_bin_ncached_get_local(&bin) ==
(i / 2 + i % 2), "Wrong ncached value"); (i / 2 + i % 2), "Wrong ncached value");
expect_true(cache_bin_nstashed_get_local(&bin, bin_info) == expect_true(cache_bin_nstashed_get_local(&bin) ==
i / 2, "Wrong nstashed value"); i / 2, "Wrong nstashed value");
if (i % 2 == 0) { if (i % 2 == 0) {
cache_bin_dalloc_easy(&bin, &ptrs[i]); cache_bin_dalloc_easy(&bin, &ptrs[i]);
@ -369,22 +360,21 @@ TEST_BEGIN(test_cache_bin_stash) {
expect_true(diff % 2 == 0, "Should be able to alloc"); expect_true(diff % 2 == 0, "Should be able to alloc");
} else { } else {
expect_false(ret, "Should not alloc stashed"); expect_false(ret, "Should not alloc stashed");
expect_true(cache_bin_nstashed_get_local(&bin, expect_true(cache_bin_nstashed_get_local(&bin) == ncached_max / 2,
bin_info) == ncached_max / 2,
"Wrong nstashed value"); "Wrong nstashed value");
} }
} }
test_bin_init(&bin, &info); test_bin_init(&bin, &info);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max, 0); do_flush_stashed_test(&bin, ptrs, ncached_max, 0);
do_flush_stashed_test(&bin, bin_info, ptrs, 0, ncached_max); do_flush_stashed_test(&bin, ptrs, 0, ncached_max);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2, do_flush_stashed_test(&bin, ptrs, ncached_max / 2,
ncached_max / 2); ncached_max / 2);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4, do_flush_stashed_test(&bin, ptrs, ncached_max / 4,
ncached_max / 2); ncached_max / 2);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2, do_flush_stashed_test(&bin, ptrs, ncached_max / 2,
ncached_max / 4); ncached_max / 4);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4, do_flush_stashed_test(&bin, ptrs, ncached_max / 4,
ncached_max / 4); ncached_max / 4);
} }
TEST_END TEST_END

4
test/unit/tcache_max.c
View File

@ -81,8 +81,7 @@ tcache_bytes_read_local(void) {
if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) { if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) {
continue; continue;
} }
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin, cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin);
&cache_bin->bin_info);
tcache_bytes += ncached * sz_index2size(i); tcache_bytes += ncached * sz_index2size(i);
} }
return tcache_bytes; return tcache_bytes;
@ -370,4 +369,3 @@ main(void) {
test_tcache_max, test_tcache_max,
test_thread_tcache_max); test_thread_tcache_max);
} }