project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleanup cache_bin_info_t* info input args

Browse Source
This commit is contained in:
Shirui Cheng 2023-10-24 13:51:14 -07:00 committed by Qi Wang
parent 3025b021b9
commit e4817c8d89
8 changed files with 175 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/jemalloc/internal/arena_externs.h
View File

@ -63,8 +63,7 @@ void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
void arena_reset(tsd_t *tsd, arena_t *arena);
void arena_destroy(tsd_t *tsd, arena_t *arena);
void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena,
cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind,
const unsigned nfill);
cache_bin_t *cache_bin, szind_t binind, const unsigned nfill);
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
szind_t ind, bool zero, bool slab);

97
include/jemalloc/internal/cache_bin.h
View File

@ -202,19 +202,19 @@ cache_bin_disabled(cache_bin_t *bin) {
return disabled;
}
/* Returns ncached_max: Upper limit on ncached. */
static inline cache_bin_sz_t
cache_bin_info_ncached_max_get(cache_bin_t *bin, cache_bin_info_t *info) {
assert(!cache_bin_disabled(bin));
assert(info == &bin->bin_info);
return info->ncached_max;
}
/* Gets ncached_max without asserting that the bin is enabled. */
static inline cache_bin_sz_t
cache_bin_ncached_max_get_unsafe(cache_bin_t *bin) {
return bin->bin_info.ncached_max;
}
/* Returns ncached_max: Upper limit on ncached. */
static inline cache_bin_sz_t
cache_bin_ncached_max_get(cache_bin_t *bin) {
assert(!cache_bin_disabled(bin));
return cache_bin_ncached_max_get_unsafe(bin);
}
/*
* Internal.
*
@ -267,9 +267,9 @@ cache_bin_ncached_get_internal(cache_bin_t *bin) {
* possible.
*/
static inline cache_bin_sz_t
cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_ncached_get_local(cache_bin_t *bin) {
cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
assert(n <= cache_bin_info_ncached_max_get(bin, info));
assert(n <= cache_bin_ncached_max_get(bin));
return n;
}
@ -304,9 +304,9 @@ cache_bin_empty_position_get(cache_bin_t *bin) {
* arena statistics collection.
*/
static inline uint16_t
cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_low_bits_low_bound_get(cache_bin_t *bin) {
return (uint16_t)bin->low_bits_empty -
cache_bin_info_ncached_max_get(bin, info) * sizeof(void *);
cache_bin_ncached_max_get(bin) * sizeof(void *);
}
/*
@ -315,8 +315,8 @@ cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
* A pointer to the position with the lowest address of the backing array.
*/
static inline void **
cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info);
cache_bin_low_bound_get(cache_bin_t *bin) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
void **ret = cache_bin_empty_position_get(bin) - ncached_max;
assert(ret <= bin->stack_head);
@ -328,8 +328,8 @@ cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
* batch fill a nonempty cache bin.
*/
static inline void
cache_bin_assert_empty(cache_bin_t *bin, cache_bin_info_t *info) {
assert(cache_bin_ncached_get_local(bin, info) == 0);
cache_bin_assert_empty(cache_bin_t *bin) {
assert(cache_bin_ncached_get_local(bin) == 0);
assert(cache_bin_empty_position_get(bin) == bin->stack_head);
}
@ -346,10 +346,10 @@ cache_bin_low_water_get_internal(cache_bin_t *bin) {
/* Returns the numeric value of low water in [0, ncached]. */
static inline cache_bin_sz_t
cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_low_water_get(cache_bin_t *bin) {
cache_bin_sz_t low_water = cache_bin_low_water_get_internal(bin);
assert(low_water <= cache_bin_info_ncached_max_get(bin, info));
assert(low_water <= cache_bin_ncached_get_local(bin, info));
assert(low_water <= cache_bin_ncached_max_get(bin));
assert(low_water <= cache_bin_ncached_get_local(bin));
cache_bin_assert_earlier(bin, (uint16_t)(uintptr_t)bin->stack_head,
bin->low_bits_low_water);
@ -530,17 +530,16 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
/* Get the number of stashed pointers. */
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info);
uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin,
info);
cache_bin_nstashed_get_internal(cache_bin_t *bin) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin);
cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound,
bin->low_bits_full) / sizeof(void *);
assert(n <= ncached_max);
if (config_debug && n != 0) {
/* Below are for assertions only. */
void **low_bound = cache_bin_low_bound_get(bin, info);
void **low_bound = cache_bin_low_bound_get(bin);
assert((uint16_t)(uintptr_t)low_bound == low_bits_low_bound);
void *stashed = *(low_bound + n - 1);
@ -556,9 +555,9 @@ cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) {
}
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin, info);
assert(n <= cache_bin_info_ncached_max_get(bin, info));
cache_bin_nstashed_get_local(cache_bin_t *bin) {
cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin);
assert(n <= cache_bin_ncached_max_get(bin));
return n;
}
@ -579,12 +578,12 @@ cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
* This function should not call other utility functions because the racy
* condition may cause unexpected / undefined behaviors in unverified utility
* functions. Currently, this function calls two utility functions
* cache_bin_info_ncached_max_get and cache_bin_low_bits_low_bound_get because
* cache_bin_ncached_max_get and cache_bin_low_bits_low_bound_get because
* they help access values that will not be concurrently modified.
*/
static inline void
cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info,
cache_bin_sz_t *ncached, cache_bin_sz_t *nstashed) {
cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_sz_t *ncached,
cache_bin_sz_t *nstashed) {
/* Racy version of cache_bin_ncached_get_internal. */
cache_bin_sz_t diff = bin->low_bits_empty -
(uint16_t)(uintptr_t)bin->stack_head;
@ -592,8 +591,7 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info,
*ncached = n;
/* Racy version of cache_bin_nstashed_get_internal. */
uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin,
info);
uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin);
n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *);
*nstashed = n;
/*
@ -643,9 +641,9 @@ struct cache_bin_ptr_array_s {
* finish_fill call before doing any alloc/dalloc operations on the bin.
*/
static inline void
cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info,
cache_bin_ptr_array_t *arr, cache_bin_sz_t nfill) {
cache_bin_assert_empty(bin, info);
cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr,
cache_bin_sz_t nfill) {
cache_bin_assert_empty(bin);
arr->ptr = cache_bin_empty_position_get(bin) - nfill;
}
@ -655,9 +653,9 @@ cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_info_t *info,
* case of OOM.
*/
static inline void
cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info,
cache_bin_ptr_array_t *arr, cache_bin_sz_t nfilled) {
cache_bin_assert_empty(bin, info);
cache_bin_finish_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr,
cache_bin_sz_t nfilled) {
cache_bin_assert_empty(bin);
void **empty_position = cache_bin_empty_position_get(bin);
if (nfilled < arr->n) {
memmove(empty_position - nfilled, empty_position - arr->n,
@ -671,17 +669,17 @@ cache_bin_finish_fill(cache_bin_t *bin, cache_bin_info_t *info,
* everything we give them.
*/
static inline void
cache_bin_init_ptr_array_for_flush(cache_bin_t *bin, cache_bin_info_t *info,
cache_bin_init_ptr_array_for_flush(cache_bin_t *bin,
cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) {
arr->ptr = cache_bin_empty_position_get(bin) - nflush;
assert(cache_bin_ncached_get_local(bin, info) == 0
assert(cache_bin_ncached_get_local(bin) == 0
|| *arr->ptr != NULL);
}
static inline void
cache_bin_finish_flush(cache_bin_t *bin, cache_bin_info_t *info,
cache_bin_ptr_array_t *arr, cache_bin_sz_t nflushed) {
unsigned rem = cache_bin_ncached_get_local(bin, info) - nflushed;
cache_bin_finish_flush(cache_bin_t *bin, cache_bin_ptr_array_t *arr,
cache_bin_sz_t nflushed) {
unsigned rem = cache_bin_ncached_get_local(bin) - nflushed;
memmove(bin->stack_head + nflushed, bin->stack_head,
rem * sizeof(void *));
bin->stack_head += nflushed;
@ -690,23 +688,22 @@ cache_bin_finish_flush(cache_bin_t *bin, cache_bin_info_t *info,
static inline void
cache_bin_init_ptr_array_for_stashed(cache_bin_t *bin, szind_t binind,
cache_bin_info_t *info, cache_bin_ptr_array_t *arr,
cache_bin_sz_t nstashed) {
cache_bin_ptr_array_t *arr, cache_bin_sz_t nstashed) {
assert(nstashed > 0);
assert(cache_bin_nstashed_get_local(bin, info) == nstashed);
assert(cache_bin_nstashed_get_local(bin) == nstashed);
void **low_bound = cache_bin_low_bound_get(bin, info);
void **low_bound = cache_bin_low_bound_get(bin);
arr->ptr = low_bound;
assert(*arr->ptr != NULL);
}
static inline void
cache_bin_finish_flush_stashed(cache_bin_t *bin, cache_bin_info_t *info) {
void **low_bound = cache_bin_low_bound_get(bin, info);
cache_bin_finish_flush_stashed(cache_bin_t *bin) {
void **low_bound = cache_bin_low_bound_get(bin);
/* Reset the bin local full position. */
bin->low_bits_full = (uint16_t)(uintptr_t)low_bound;
assert(cache_bin_nstashed_get_local(bin, info) == 0);
assert(cache_bin_nstashed_get_local(bin) == 0);
}
/*

7
include/jemalloc/internal/tcache_inlines.h
View File

@ -200,8 +200,7 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
arena_dalloc_small(tsd_tsdn(tsd), ptr);
return;
}
cache_bin_sz_t max = cache_bin_info_ncached_max_get(
bin, &bin->bin_info);
cache_bin_sz_t max = cache_bin_ncached_max_get(bin);
unsigned remain = max >> opt_lg_tcache_flush_small_div;
tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr);
@ -221,8 +220,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
cache_bin_t *bin = &tcache->bins[binind];
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
unsigned remain = cache_bin_info_ncached_max_get(
bin, &bin->bin_info) >> opt_lg_tcache_flush_large_div;
unsigned remain = cache_bin_ncached_max_get(bin) >>
opt_lg_tcache_flush_large_div;
tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret);

13
src/arena.c
View File

@ -168,8 +168,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
}
cache_bin_sz_t ncached, nstashed;
cache_bin_nitems_get_remote(cache_bin,
&cache_bin->bin_info, &ncached, &nstashed);
cache_bin_nitems_get_remote(cache_bin, &ncached, &nstashed);
astats->tcache_bytes += ncached * sz_index2size(i);
astats->tcache_stashed_bytes += nstashed *
sz_index2size(i);
@ -1020,16 +1019,14 @@ arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
void
arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena,
cache_bin_t *cache_bin, cache_bin_info_t *cache_bin_info, szind_t binind,
const unsigned nfill) {
assert(cache_bin_ncached_get_local(cache_bin, cache_bin_info) == 0);
cache_bin_t *cache_bin, szind_t binind, const unsigned nfill) {
assert(cache_bin_ncached_get_local(cache_bin) == 0);
assert(nfill != 0);
const bin_info_t *bin_info = &bin_infos[binind];
CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nfill);
cache_bin_init_ptr_array_for_fill(cache_bin, cache_bin_info, &ptrs,
nfill);
cache_bin_init_ptr_array_for_fill(cache_bin, &ptrs, nfill);
/*
* Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull
* slabs. After both are exhausted, new slabs will be allocated through
@ -1143,7 +1140,7 @@ label_refill:
fresh_slab = NULL;
}
cache_bin_finish_fill(cache_bin, cache_bin_info, &ptrs, filled);
cache_bin_finish_fill(cache_bin, &ptrs, filled);
arena_decay_tick(tsdn, arena);
}

2
src/cache_bin.c
View File

@ -100,7 +100,7 @@ cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head);
assert(free_spots == bin_stack_size);
if (!cache_bin_disabled(bin)) {
assert(cache_bin_ncached_get_local(bin, &bin->bin_info) == 0);
assert(cache_bin_ncached_get_local(bin) == 0);
}
assert(cache_bin_empty_position_get(bin) == empty_position);

56
src/tcache.c
View File

@ -142,10 +142,8 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
cache_bin_t *cache_bin = &tcache->bins[szind];
assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow));
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin);
assert(!tcache_slow->bin_refilled[szind]);
size_t nflush = low_water - (low_water >> 2);
@ -168,8 +166,8 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
* Reduce fill count by 2X. Limit lg_fill_div such that
* the fill count is always at least 1.
*/
if ((cache_bin_info_ncached_max_get(cache_bin, &cache_bin->bin_info)
>> (tcache_slow->lg_fill_div[szind] + 1)) >= 1) {
if ((cache_bin_ncached_max_get(cache_bin) >>
(tcache_slow->lg_fill_div[szind] + 1)) >= 1) {
tcache_slow->lg_fill_div[szind]++;
}
}
@ -181,10 +179,8 @@ tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
assert(szind >= SC_NBINS);
cache_bin_t *cache_bin = &tcache->bins[szind];
assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow));
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin);
tcache_bin_flush_large(tsd, tcache, cache_bin, szind,
(unsigned)(ncached - low_water + (low_water >> 2)));
}
@ -206,8 +202,7 @@ tcache_event(tsd_t *tsd) {
}
tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin);
if (low_water > 0) {
if (is_small) {
tcache_gc_small(tsd, tcache_slow, tcache, szind);
@ -255,13 +250,12 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena,
assert(tcache_slow->arena != NULL);
assert(!tcache_bin_disabled(binind, cache_bin, tcache_slow));
unsigned nfill = cache_bin_info_ncached_max_get(cache_bin,
&cache_bin->bin_info) >> tcache_slow->lg_fill_div[binind];
unsigned nfill = cache_bin_ncached_max_get(cache_bin)
>> tcache_slow->lg_fill_div[binind];
if (nfill == 0) {
nfill = 1;
}
arena_cache_bin_fill_small(tsdn, arena, cache_bin,
&cache_bin->bin_info, binind, nfill);
arena_cache_bin_fill_small(tsdn, arena, cache_bin, binind, nfill);
tcache_slow->bin_refilled[binind] = true;
ret = cache_bin_alloc(cache_bin, tcache_success);
@ -533,20 +527,17 @@ tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow));
tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small);
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin);
assert((cache_bin_sz_t)rem <= ncached);
unsigned nflush = ncached - rem;
CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nflush);
cache_bin_init_ptr_array_for_flush(cache_bin, &cache_bin->bin_info,
&ptrs, nflush);
cache_bin_init_ptr_array_for_flush(cache_bin, &ptrs, nflush);
tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nflush,
small);
cache_bin_finish_flush(cache_bin, &cache_bin->bin_info, &ptrs,
ncached - rem);
cache_bin_finish_flush(cache_bin, &ptrs, ncached - rem);
}
void
@ -575,33 +566,30 @@ void
tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
szind_t binind, bool is_small) {
assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow));
cache_bin_info_t *info = &cache_bin->bin_info;
/*
* The two below are for assertion only. The content of original cached
* items remain unchanged -- the stashed items reside on the other end
* of the stack. Checking the stack head and ncached to verify.
*/
void *head_content = *cache_bin->stack_head;
cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin,
info);
cache_bin_sz_t orig_cached = cache_bin_ncached_get_local(cache_bin);
cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin, info);
assert(orig_cached + nstashed <=
cache_bin_info_ncached_max_get(cache_bin, info));
cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin);
assert(orig_cached + nstashed <= cache_bin_ncached_max_get(cache_bin));
if (nstashed == 0) {
return;
}
CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nstashed);
cache_bin_init_ptr_array_for_stashed(cache_bin, binind, info, &ptrs,
cache_bin_init_ptr_array_for_stashed(cache_bin, binind, &ptrs,
nstashed);
san_check_stashed_ptrs(ptrs.ptr, nstashed, sz_index2size(binind));
tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nstashed,
is_small);
cache_bin_finish_flush_stashed(cache_bin, info);
cache_bin_finish_flush_stashed(cache_bin);
assert(cache_bin_nstashed_get_local(cache_bin, info) == 0);
assert(cache_bin_ncached_get_local(cache_bin, info) == orig_cached);
assert(cache_bin_nstashed_get_local(cache_bin) == 0);
assert(cache_bin_ncached_get_local(cache_bin) == orig_cached);
assert(head_content == *cache_bin->stack_head);
}
@ -633,7 +621,7 @@ tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size,
cache_bin_t *bin = &tcache->bins[bin_ind];
*ncached_max = tcache_bin_disabled(bin_ind, bin, tcache->tcache_slow) ?
0: cache_bin_info_ncached_max_get(bin, &bin->bin_info);
0: cache_bin_ncached_max_get(bin);
return false;
}
@ -1105,7 +1093,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
if (tsd_tcache) {
cache_bin_t *cache_bin = &tcache->bins[0];
cache_bin_assert_empty(cache_bin, &cache_bin->bin_info);
cache_bin_assert_empty(cache_bin);
}
if (tsd_tcache && cache_bin_stack_use_thp()) {
b0_dalloc_tcache_stack(tsd_tsdn(tsd), tcache_slow->dyn_alloc);

200
test/unit/cache_bin.c
View File

@ -1,19 +1,18 @@
#include "test/jemalloc_test.h"
static void
do_fill_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
cache_bin_sz_t ncached_max, cache_bin_sz_t nfill_attempt,
cache_bin_sz_t nfill_succeed) {
do_fill_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t ncached_max,
cache_bin_sz_t nfill_attempt, cache_bin_sz_t nfill_succeed) {
bool success;
void *ptr;
assert_true(cache_bin_ncached_get_local(bin, info) == 0, "");
assert_true(cache_bin_ncached_get_local(bin) == 0, "");
CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill_attempt);
cache_bin_init_ptr_array_for_fill(bin, info, &arr, nfill_attempt);
cache_bin_init_ptr_array_for_fill(bin, &arr, nfill_attempt);
for (cache_bin_sz_t i = 0; i < nfill_succeed; i++) {
arr.ptr[i] = &ptrs[i];
}
cache_bin_finish_fill(bin, info, &arr, nfill_succeed);
expect_true(cache_bin_ncached_get_local(bin, info) == nfill_succeed,
cache_bin_finish_fill(bin, &arr, nfill_succeed);
expect_true(cache_bin_ncached_get_local(bin) == nfill_succeed,
"");
cache_bin_low_water_set(bin);
@ -22,18 +21,18 @@ do_fill_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
expect_true(success, "");
expect_ptr_eq(ptr, (void *)&ptrs[i],
"Should pop in order filled");
expect_true(cache_bin_low_water_get(bin, info)
expect_true(cache_bin_low_water_get(bin)
== nfill_succeed - i - 1, "");
}
expect_true(cache_bin_ncached_get_local(bin, info) == 0, "");
expect_true(cache_bin_low_water_get(bin, info) == 0, "");
expect_true(cache_bin_ncached_get_local(bin) == 0, "");
expect_true(cache_bin_low_water_get(bin) == 0, "");
}
static void
do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
cache_bin_sz_t nfill, cache_bin_sz_t nflush) {
do_flush_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
cache_bin_sz_t nflush) {
bool success;
assert_true(cache_bin_ncached_get_local(bin, info) == 0, "");
assert_true(cache_bin_ncached_get_local(bin) == 0, "");
for (cache_bin_sz_t i = 0; i < nfill; i++) {
success = cache_bin_dalloc_easy(bin, &ptrs[i]);
@ -41,30 +40,30 @@ do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
}
CACHE_BIN_PTR_ARRAY_DECLARE(arr, nflush);
cache_bin_init_ptr_array_for_flush(bin, info, &arr, nflush);
cache_bin_init_ptr_array_for_flush(bin, &arr, nflush);
for (cache_bin_sz_t i = 0; i < nflush; i++) {
expect_ptr_eq(arr.ptr[i], &ptrs[nflush - i - 1], "");
}
cache_bin_finish_flush(bin, info, &arr, nflush);
cache_bin_finish_flush(bin, &arr, nflush);
expect_true(cache_bin_ncached_get_local(bin, info) == nfill - nflush,
expect_true(cache_bin_ncached_get_local(bin) == nfill - nflush,
"");
while (cache_bin_ncached_get_local(bin, info) > 0) {
while (cache_bin_ncached_get_local(bin) > 0) {
cache_bin_alloc(bin, &success);
}
}
static void
do_batch_alloc_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
cache_bin_sz_t nfill, size_t batch) {
assert_true(cache_bin_ncached_get_local(bin, info) == 0, "");
do_batch_alloc_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
size_t batch) {
assert_true(cache_bin_ncached_get_local(bin) == 0, "");
CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill);
cache_bin_init_ptr_array_for_fill(bin, info, &arr, nfill);
cache_bin_init_ptr_array_for_fill(bin, &arr, nfill);
for (cache_bin_sz_t i = 0; i < nfill; i++) {
arr.ptr[i] = &ptrs[i];
}
cache_bin_finish_fill(bin, info, &arr, nfill);
assert_true(cache_bin_ncached_get_local(bin, info) == nfill, "");
cache_bin_finish_fill(bin, &arr, nfill);
assert_true(cache_bin_ncached_get_local(bin) == nfill, "");
cache_bin_low_water_set(bin);
void **out = malloc((batch + 1) * sizeof(void *));
@ -73,9 +72,9 @@ do_batch_alloc_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
for (cache_bin_sz_t i = 0; i < (cache_bin_sz_t)n; i++) {
expect_ptr_eq(out[i], &ptrs[i], "");
}
expect_true(cache_bin_low_water_get(bin, info) == nfill -
expect_true(cache_bin_low_water_get(bin) == nfill -
(cache_bin_sz_t)n, "");
while (cache_bin_ncached_get_local(bin, info) > 0) {
while (cache_bin_ncached_get_local(bin) > 0) {
bool success;
cache_bin_alloc(bin, &success);
}
@ -106,13 +105,11 @@ TEST_BEGIN(test_cache_bin) {
cache_bin_info_init(&info, ncached_max);
cache_bin_t bin;
test_bin_init(&bin, &info);
cache_bin_info_t *bin_info = &bin.bin_info;
/* Initialize to empty; should then have 0 elements. */
expect_d_eq(ncached_max, cache_bin_info_ncached_max_get(&bin,
&bin.bin_info), "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, "");
expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, "");
expect_d_eq(ncached_max, cache_bin_ncached_max_get(&bin), "");
expect_true(cache_bin_ncached_get_local(&bin) == 0, "");
expect_true(cache_bin_low_water_get(&bin) == 0, "");
ptr = cache_bin_alloc_easy(&bin, &success);
expect_false(success, "Shouldn't successfully allocate when empty");
@ -129,14 +126,14 @@ TEST_BEGIN(test_cache_bin) {
void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0);
assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == i, "");
expect_true(cache_bin_ncached_get_local(&bin) == i, "");
success = cache_bin_dalloc_easy(&bin, &ptrs[i]);
expect_true(success,
"Should be able to dalloc into a non-full cache bin.");
expect_true(cache_bin_low_water_get(&bin, bin_info) == 0,
expect_true(cache_bin_low_water_get(&bin) == 0,
"Pushes and pops shouldn't change low water of zero.");
}
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max,
expect_true(cache_bin_ncached_get_local(&bin) == ncached_max,
"");
success = cache_bin_dalloc_easy(&bin, &ptrs[ncached_max]);
expect_false(success, "Shouldn't be able to dalloc into a full bin.");
@ -144,9 +141,9 @@ TEST_BEGIN(test_cache_bin) {
cache_bin_low_water_set(&bin);
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_low_water_get(&bin, bin_info)
expect_true(cache_bin_low_water_get(&bin)
== ncached_max - i, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
expect_true(cache_bin_ncached_get_local(&bin)
== ncached_max - i, "");
/*
* This should fail -- the easy variant can't change the low
@ -155,9 +152,9 @@ TEST_BEGIN(test_cache_bin) {
ptr = cache_bin_alloc_easy(&bin, &success);
expect_ptr_null(ptr, "");
expect_false(success, "");
expect_true(cache_bin_low_water_get(&bin, bin_info)
expect_true(cache_bin_low_water_get(&bin)
== ncached_max - i, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
expect_true(cache_bin_ncached_get_local(&bin)
== ncached_max - i, "");
/* This should succeed, though. */
@ -165,13 +162,13 @@ TEST_BEGIN(test_cache_bin) {
expect_true(success, "");
expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1],
"Alloc should pop in stack order");
expect_true(cache_bin_low_water_get(&bin, bin_info)
expect_true(cache_bin_low_water_get(&bin)
== ncached_max - i - 1, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
expect_true(cache_bin_ncached_get_local(&bin)
== ncached_max - i - 1, "");
}
/* Now we're empty -- all alloc attempts should fail. */
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, "");
expect_true(cache_bin_ncached_get_local(&bin) == 0, "");
ptr = cache_bin_alloc_easy(&bin, &success);
expect_ptr_null(ptr, "");
expect_false(success, "");
@ -187,7 +184,7 @@ TEST_BEGIN(test_cache_bin) {
for (cache_bin_sz_t i = ncached_max / 2; i < ncached_max; i++) {
cache_bin_dalloc_easy(&bin, &ptrs[i]);
}
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max,
expect_true(cache_bin_ncached_get_local(&bin) == ncached_max,
"");
for (cache_bin_sz_t i = ncached_max - 1; i >= ncached_max / 2; i--) {
/*
@ -204,77 +201,72 @@ TEST_BEGIN(test_cache_bin) {
expect_ptr_null(ptr, "");
/* We're going to test filling -- we must be empty to start. */
while (cache_bin_ncached_get_local(&bin, bin_info)) {
while (cache_bin_ncached_get_local(&bin)) {
cache_bin_alloc(&bin, &success);
expect_true(success, "");
}
/* Test fill. */
/* Try to fill all, succeed fully. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max,
do_fill_test(&bin, ptrs, ncached_max, ncached_max,
ncached_max);
/* Try to fill all, succeed partially. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max,
do_fill_test(&bin, ptrs, ncached_max, ncached_max,
ncached_max / 2);
/* Try to fill all, fail completely. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, 0);
do_fill_test(&bin, ptrs, ncached_max, ncached_max, 0);
/* Try to fill some, succeed fully. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2,
do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2,
ncached_max / 2);
/* Try to fill some, succeed partially. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2,
do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2,
ncached_max / 4);
/* Try to fill some, fail completely. */
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, 0);
do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2, 0);
do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max);
do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2);
do_flush_test(&bin, bin_info, ptrs, ncached_max, 0);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, 0);
do_flush_test(&bin, ptrs, ncached_max, ncached_max);
do_flush_test(&bin, ptrs, ncached_max, ncached_max / 2);
do_flush_test(&bin, ptrs, ncached_max, 0);
do_flush_test(&bin, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_test(&bin, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_test(&bin, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max,
ncached_max * 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max,
ncached_max / 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 4);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 0);
do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max);
do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max * 2);
do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max / 2);
do_batch_alloc_test(&bin, ptrs, ncached_max, 2);
do_batch_alloc_test(&bin, ptrs, ncached_max, 1);
do_batch_alloc_test(&bin, ptrs, ncached_max, 0);
do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max / 2);
do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max);
do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max / 4);
do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 2);
do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 1);
do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, ptrs, 2, ncached_max);
do_batch_alloc_test(&bin, ptrs, 2, 2);
do_batch_alloc_test(&bin, ptrs, 2, 1);
do_batch_alloc_test(&bin, ptrs, 2, 0);
do_batch_alloc_test(&bin, ptrs, 1, 2);
do_batch_alloc_test(&bin, ptrs, 1, 1);
do_batch_alloc_test(&bin, ptrs, 1, 0);
do_batch_alloc_test(&bin, ptrs, 0, 2);
do_batch_alloc_test(&bin, ptrs, 0, 1);
do_batch_alloc_test(&bin, ptrs, 0, 0);
free(ptrs);
}
TEST_END
static void
do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
cache_bin_sz_t nfill, cache_bin_sz_t nstash) {
expect_true(cache_bin_ncached_get_local(bin, info) == 0,
do_flush_stashed_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
cache_bin_sz_t nstash) {
expect_true(cache_bin_ncached_get_local(bin) == 0,
"Bin not empty");
expect_true(cache_bin_nstashed_get_local(bin, info) == 0,
expect_true(cache_bin_nstashed_get_local(bin) == 0,
"Bin not empty");
expect_true(nfill + nstash <= info->ncached_max, "Exceeded max");
expect_true(nfill + nstash <= bin->bin_info.ncached_max, "Exceeded max");
bool ret;
/* Fill */
@ -282,7 +274,7 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
ret = cache_bin_dalloc_easy(bin, &ptrs[i]);
expect_true(ret, "Unexpected fill failure");
}
expect_true(cache_bin_ncached_get_local(bin, info) == nfill,
expect_true(cache_bin_ncached_get_local(bin) == nfill,
"Wrong cached count");
/* Stash */
@ -290,10 +282,10 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
ret = cache_bin_stash(bin, &ptrs[i + nfill]);
expect_true(ret, "Unexpected stash failure");
}
expect_true(cache_bin_nstashed_get_local(bin, info) == nstash,
expect_true(cache_bin_nstashed_get_local(bin) == nstash,
"Wrong stashed count");
if (nfill + nstash == info->ncached_max) {
if (nfill + nstash == bin->bin_info.ncached_max) {
ret = cache_bin_dalloc_easy(bin, &ptrs[0]);
expect_false(ret, "Should not dalloc into a full bin");
ret = cache_bin_stash(bin, &ptrs[0]);
@ -308,19 +300,19 @@ do_flush_stashed_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
expect_true((uintptr_t)ptr < (uintptr_t)&ptrs[nfill],
"Should not alloc stashed ptrs");
}
expect_true(cache_bin_ncached_get_local(bin, info) == 0,
expect_true(cache_bin_ncached_get_local(bin) == 0,
"Wrong cached count");
expect_true(cache_bin_nstashed_get_local(bin, info) == nstash,
expect_true(cache_bin_nstashed_get_local(bin) == nstash,
"Wrong stashed count");
cache_bin_alloc(bin, &ret);
expect_false(ret, "Should not alloc stashed");
/* Clear stashed ones */
cache_bin_finish_flush_stashed(bin, info);
expect_true(cache_bin_ncached_get_local(bin, info) == 0,
cache_bin_finish_flush_stashed(bin);
expect_true(cache_bin_ncached_get_local(bin) == 0,
"Wrong cached count");
expect_true(cache_bin_nstashed_get_local(bin, info) == 0,
expect_true(cache_bin_nstashed_get_local(bin) == 0,
"Wrong stashed count");
cache_bin_alloc(bin, &ret);
@ -334,7 +326,6 @@ TEST_BEGIN(test_cache_bin_stash) {
cache_bin_info_t info;
cache_bin_info_init(&info, ncached_max);
test_bin_init(&bin, &info);
cache_bin_info_t *bin_info = &bin.bin_info;
/*
* The content of this array is not accessed; instead the interior
@ -344,9 +335,9 @@ TEST_BEGIN(test_cache_bin_stash) {
assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
bool ret;
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_ncached_get_local(&bin, bin_info) ==
expect_true(cache_bin_ncached_get_local(&bin) ==
(i / 2 + i % 2), "Wrong ncached value");
expect_true(cache_bin_nstashed_get_local(&bin, bin_info) ==
expect_true(cache_bin_nstashed_get_local(&bin) ==
i / 2, "Wrong nstashed value");
if (i % 2 == 0) {
cache_bin_dalloc_easy(&bin, &ptrs[i]);
@ -369,22 +360,21 @@ TEST_BEGIN(test_cache_bin_stash) {
expect_true(diff % 2 == 0, "Should be able to alloc");
} else {
expect_false(ret, "Should not alloc stashed");
expect_true(cache_bin_nstashed_get_local(&bin,
bin_info) == ncached_max / 2,
expect_true(cache_bin_nstashed_get_local(&bin) == ncached_max / 2,
"Wrong nstashed value");
}
}
test_bin_init(&bin, &info);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max, 0);
do_flush_stashed_test(&bin, bin_info, ptrs, 0, ncached_max);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2,
do_flush_stashed_test(&bin, ptrs, ncached_max, 0);
do_flush_stashed_test(&bin, ptrs, 0, ncached_max);
do_flush_stashed_test(&bin, ptrs, ncached_max / 2,
ncached_max / 2);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4,
do_flush_stashed_test(&bin, ptrs, ncached_max / 4,
ncached_max / 2);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2,
do_flush_stashed_test(&bin, ptrs, ncached_max / 2,
ncached_max / 4);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4,
do_flush_stashed_test(&bin, ptrs, ncached_max / 4,
ncached_max / 4);
}
TEST_END

4
test/unit/tcache_max.c
View File

@ -81,8 +81,7 @@ tcache_bytes_read_local(void) {
if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) {
continue;
}
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin);
tcache_bytes += ncached * sz_index2size(i);
}
return tcache_bytes;
@ -370,4 +369,3 @@ main(void) {
test_tcache_max,
test_thread_tcache_max);
}