project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pre-generate ncached_max for all bins for better tcache_max tuning experience.

Browse Source
This commit is contained in:
guangli-dai 2023-08-22 16:31:54 -07:00 committed by Qi Wang
parent 36becb1302
commit 6b197fdd46
12 changed files with 417 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/jemalloc/internal/arena_inlines_b.h
View File

@ -198,7 +198,9 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
assert(sz_can_use_slab(size));
return tcache_alloc_small(tsdn_tsd(tsdn), arena,
tcache, size, ind, zero, slow_path);
} else if (likely(size <= tcache_max_get(tcache))) {
} else if (likely(ind < TCACHE_NBINS_MAX &&
!tcache_bin_disabled(ind, &tcache->bins[ind],
tcache->tcache_slow))) {
return tcache_alloc_large(tsdn_tsd(tsdn), arena,
tcache, size, ind, zero, slow_path);
}
@ -298,7 +300,9 @@ JEMALLOC_ALWAYS_INLINE void
arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
bool slow_path) {
assert (!tsdn_null(tsdn) && tcache != NULL);
if (szind < tcache_nhbins_get(tcache)) {
if (szind < TCACHE_NBINS_MAX &&
!tcache_bin_disabled(szind, &tcache->bins[szind],
tcache->tcache_slow)) {
if (config_prof && unlikely(szind < SC_NBINS)) {
arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
} else {

77
include/jemalloc/internal/cache_bin.h
View File

@ -23,16 +23,20 @@
*/
typedef uint16_t cache_bin_sz_t;
#define JUNK_ADDR ((uintptr_t)0x7a7a7a7a7a7a7a7aULL)
/*
* Leave a noticeable mark pattern on the cache bin stack boundaries, in case a
* bug starts leaking those. Make it look like the junk pattern but be distinct
* from it.
*/
static const uintptr_t cache_bin_preceding_junk =
(uintptr_t)0x7a7a7a7a7a7a7a7aULL;
/* Note: a7 vs. 7a above -- this tells you which pointer leaked. */
static const uintptr_t cache_bin_trailing_junk =
(uintptr_t)0xa7a7a7a7a7a7a7a7ULL;
static const uintptr_t cache_bin_preceding_junk = JUNK_ADDR;
/* Note: JUNK_ADDR vs. JUNK_ADDR + 1 -- this tells you which pointer leaked. */
static const uintptr_t cache_bin_trailing_junk = JUNK_ADDR + 1;
/*
* A pointer used to initialize a fake stack_head for disabled small bins
* so that the enabled/disabled assessment does not rely on ncached_max.
*/
extern const uintptr_t disabled_bin;
/*
* That implies the following value, for the maximum number of items in any
@ -174,9 +178,35 @@ cache_bin_nonfast_aligned(const void *ptr) {
return ((uintptr_t)ptr & san_cache_bin_nonfast_mask) == 0;
}
static inline const void *
cache_bin_disabled_bin_stack(void) {
return &disabled_bin;
}
/*
* If a cache bin was zero initialized (either because it lives in static or
* thread-local storage, or was memset to 0), this function indicates whether or
* not cache_bin_init was called on it.
*/
static inline bool
cache_bin_still_zero_initialized(cache_bin_t *bin) {
return bin->stack_head == NULL;
}
static inline bool
cache_bin_disabled(cache_bin_t *bin) {
bool disabled = (bin->stack_head == cache_bin_disabled_bin_stack());
if (disabled) {
assert((uintptr_t)(*bin->stack_head) == JUNK_ADDR);
}
return disabled;
}
/* Returns ncached_max: Upper limit on ncached. */
static inline cache_bin_sz_t
cache_bin_info_ncached_max(cache_bin_info_t *info) {
cache_bin_info_ncached_max_get(cache_bin_t *bin, cache_bin_info_t *info) {
assert(!cache_bin_disabled(bin));
assert(info == &bin->bin_info);
return info->ncached_max;
}
@ -234,7 +264,7 @@ cache_bin_ncached_get_internal(cache_bin_t *bin) {
static inline cache_bin_sz_t
cache_bin_ncached_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
assert(n <= cache_bin_info_ncached_max(info));
assert(n <= cache_bin_info_ncached_max_get(bin, info));
return n;
}
@ -271,7 +301,7 @@ cache_bin_empty_position_get(cache_bin_t *bin) {
static inline uint16_t
cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
return (uint16_t)bin->low_bits_empty -
info->ncached_max * sizeof(void *);
cache_bin_info_ncached_max_get(bin, info) * sizeof(void *);
}
/*
@ -281,7 +311,7 @@ cache_bin_low_bits_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
*/
static inline void **
cache_bin_low_bound_get(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(info);
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info);
void **ret = cache_bin_empty_position_get(bin) - ncached_max;
assert(ret <= bin->stack_head);
@ -313,7 +343,7 @@ cache_bin_low_water_get_internal(cache_bin_t *bin) {
static inline cache_bin_sz_t
cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_sz_t low_water = cache_bin_low_water_get_internal(bin);
assert(low_water <= cache_bin_info_ncached_max(info));
assert(low_water <= cache_bin_info_ncached_max_get(bin, info));
assert(low_water <= cache_bin_ncached_get_local(bin, info));
cache_bin_assert_earlier(bin, (uint16_t)(uintptr_t)bin->stack_head,
@ -328,11 +358,13 @@ cache_bin_low_water_get(cache_bin_t *bin, cache_bin_info_t *info) {
*/
static inline void
cache_bin_low_water_set(cache_bin_t *bin) {
assert(!cache_bin_disabled(bin));
bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head;
}
static inline void
cache_bin_low_water_adjust(cache_bin_t *bin) {
assert(!cache_bin_disabled(bin));
if (cache_bin_ncached_get_internal(bin)
< cache_bin_low_water_get_internal(bin)) {
cache_bin_low_water_set(bin);
@ -494,14 +526,14 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
/* Get the number of stashed pointers. */
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(info);
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info);
uint16_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin,
info);
cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound,
bin->low_bits_full) / sizeof(void *);
assert(n <= ncached_max);
if (config_debug && n != 0) {
/* Below are for assertions only. */
void **low_bound = cache_bin_low_bound_get(bin, info);
@ -512,7 +544,8 @@ cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) {
/* Allow arbitrary pointers to be stashed in tests. */
aligned = true;
#endif
assert(n == 0 || (stashed != NULL && aligned));
assert(stashed != NULL && aligned);
}
return n;
}
@ -520,7 +553,7 @@ cache_bin_nstashed_get_internal(cache_bin_t *bin, cache_bin_info_t *info) {
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
cache_bin_sz_t n = cache_bin_nstashed_get_internal(bin, info);
assert(n <= cache_bin_info_ncached_max(info));
assert(n <= cache_bin_info_ncached_max_get(bin, info));
return n;
}
@ -541,8 +574,8 @@ cache_bin_nstashed_get_local(cache_bin_t *bin, cache_bin_info_t *info) {
* This function should not call other utility functions because the racy
* condition may cause unexpected / undefined behaviors in unverified utility
* functions. Currently, this function calls two utility functions
* cache_bin_info_ncached_max and cache_bin_low_bits_low_bound_get because they
* help access values that will not be concurrently modified.
* cache_bin_info_ncached_max_get and cache_bin_low_bits_low_bound_get because
* they help access values that will not be concurrently modified.
*/
static inline void
cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info,
@ -552,7 +585,8 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info,
(uint16_t)(uintptr_t)bin->stack_head;
cache_bin_sz_t n = diff / sizeof(void *);
assert(n <= cache_bin_info_ncached_max(info));
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max_get(bin, info);
assert(n <= ncached_max);
*ncached = n;
/* Racy version of cache_bin_nstashed_get_internal. */
@ -560,7 +594,7 @@ cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_info_t *info,
info);
n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *);
assert(n <= cache_bin_info_ncached_max(info));
assert(n <= ncached_max);
*nstashed = n;
/* Note that cannot assert ncached + nstashed <= ncached_max (racy). */
}
@ -697,13 +731,8 @@ void cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos,
void cache_bin_postincrement(void *alloc, size_t *cur_offset);
void cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc,
size_t *cur_offset);
void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max);
/*
* If a cache bin was zero initialized (either because it lives in static or
* thread-local storage, or was memset to 0), this function indicates whether or
* not cache_bin_init was called on it.
*/
bool cache_bin_still_zero_initialized(cache_bin_t *bin);
bool cache_bin_stack_use_thp(void);
#endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */

7
include/jemalloc/internal/tcache_externs.h
View File

@ -24,9 +24,9 @@ extern unsigned opt_lg_tcache_flush_large_div;
* large-object bins. This is only used during threads initialization and
* changing it will not reflect on initialized threads as expected. Thus,
* it should not be changed on the fly. To change the number of tcache bins
* in use, refer to tcache_nhbins of each tcache.
* in use, refer to tcache_nbins of each tcache.
*/
extern unsigned global_do_not_change_nhbins;
extern unsigned global_do_not_change_nbins;
/*
* Maximum cached size class. Same as above, this is only used during threads
@ -58,6 +58,7 @@ void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
void tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
tcache_t *tcache, arena_t *arena);
tcache_t *tcache_create_explicit(tsd_t *tsd);
void thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
void tcache_cleanup(tsd_t *tsd);
void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
@ -70,8 +71,8 @@ void tcache_prefork(tsdn_t *tsdn);
void tcache_postfork_parent(tsdn_t *tsdn);
void tcache_postfork_child(tsdn_t *tsdn);
void tcache_flush(tsd_t *tsd);
bool tsd_tcache_data_init(tsd_t *tsd, arena_t *arena);
bool tsd_tcache_enabled_data_init(tsd_t *tsd);
void tcache_enabled_set(tsd_t *tsd, bool enabled);
void tcache_assert_initialized(tcache_t *tcache);

142
include/jemalloc/internal/tcache_inlines.h
View File

@ -18,94 +18,72 @@ tcache_enabled_get(tsd_t *tsd) {
return tsd_tcache_enabled_get(tsd);
}
static inline void
tcache_enabled_set(tsd_t *tsd, bool enabled) {
bool was_enabled = tsd_tcache_enabled_get(tsd);
if (!was_enabled && enabled) {
tsd_tcache_data_init(tsd, NULL);
} else if (was_enabled && !enabled) {
tcache_cleanup(tsd);
}
/* Commit the state last. Above calls check current state. */
tsd_tcache_enabled_set(tsd, enabled);
tsd_slow_update(tsd);
}
static inline unsigned
tcache_nhbins_get(tcache_t *tcache) {
assert(tcache != NULL);
assert(tcache->tcache_nhbins <= TCACHE_NBINS_MAX);
return tcache->tcache_nhbins;
tcache_nbins_get(tcache_slow_t *tcache_slow) {
assert(tcache_slow != NULL);
unsigned nbins = tcache_slow->tcache_nbins;
assert(nbins <= TCACHE_NBINS_MAX);
return nbins;
}
static inline size_t
tcache_max_get(tcache_t *tcache) {
assert(tcache != NULL);
assert(tcache->tcache_max <= TCACHE_MAXCLASS_LIMIT);
return tcache->tcache_max;
tcache_max_get(tcache_slow_t *tcache_slow) {
assert(tcache_slow != NULL);
size_t tcache_max = sz_index2size(tcache_nbins_get(tcache_slow) - 1);
assert(tcache_max <= TCACHE_MAXCLASS_LIMIT);
return tcache_max;
}
static inline void
tcache_max_and_nhbins_set(tcache_t *tcache, size_t tcache_max) {
assert(tcache != NULL);
tcache_max_set(tcache_slow_t *tcache_slow, size_t tcache_max) {
assert(tcache_slow != NULL);
assert(tcache_max <= TCACHE_MAXCLASS_LIMIT);
tcache->tcache_max = tcache_max;
tcache->tcache_nhbins = sz_size2index(tcache_max) + 1;
tcache_slow->tcache_nbins = sz_size2index(tcache_max) + 1;
}
static inline void
thread_tcache_max_and_nhbins_set(tsd_t *tsd, size_t tcache_max) {
assert(tcache_max <= TCACHE_MAXCLASS_LIMIT);
assert(tcache_max == sz_s2u(tcache_max));
tcache_t *tcache = tsd_tcachep_get(tsd);
tcache_slow_t *tcache_slow;
assert(tcache != NULL);
bool enabled = tcache_available(tsd);
arena_t *assigned_arena;
if (enabled) {
tcache_slow = tcache_slow_get(tsd);
assert(tcache != NULL && tcache_slow != NULL);
assigned_arena = tcache_slow->arena;
/* Shutdown and reboot the tcache for a clean slate. */
tcache_cleanup(tsd);
tcache_bin_settings_backup(tcache_t *tcache,
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) {
cache_bin_info_init(&tcache_bin_info[i],
tcache->bins[i].bin_info.ncached_max);
}
}
JEMALLOC_ALWAYS_INLINE bool
tcache_bin_disabled(szind_t ind, cache_bin_t *bin,
tcache_slow_t *tcache_slow) {
assert(bin != NULL);
bool disabled = cache_bin_disabled(bin);
/*
* Still set tcache_max and tcache_nhbins of the tcache even if
* the tcache is not available yet because the values are
* stored in tsd_t and are always available for changing.
* If a bin's ind >= nbins or ncached_max == 0, it must be disabled.
* However, when ind < nbins, it could be either enabled
* (ncached_max > 0) or disabled (ncached_max == 0). Similarly, when
* ncached_max > 0, it could be either enabled (ind < nbins) or
* disabled (ind >= nbins). Thus, if a bin is disabled, it has either
* ind >= nbins or ncached_max == 0. If a bin is enabled, it has
* ind < nbins and ncached_max > 0.
*/
tcache_max_and_nhbins_set(tcache, tcache_max);
if (enabled) {
tsd_tcache_data_init(tsd, assigned_arena);
unsigned nbins = tcache_nbins_get(tcache_slow);
cache_bin_sz_t ncached_max = bin->bin_info.ncached_max;
if (ind >= nbins) {
assert(disabled);
} else {
assert(!disabled || ncached_max == 0);
}
if (ncached_max == 0) {
assert(disabled);
} else {
assert(!disabled || ind >= nbins);
}
if (disabled) {
assert(ind >= nbins || ncached_max == 0);
} else {
assert(ind < nbins && ncached_max > 0);
}
assert(tcache_nhbins_get(tcache) == sz_size2index(tcache_max) + 1);
}
JEMALLOC_ALWAYS_INLINE bool
tcache_small_bin_disabled(szind_t ind, cache_bin_t *bin) {
assert(ind < SC_NBINS);
assert(bin != NULL);
bool ret = cache_bin_info_ncached_max(&bin->bin_info) == 0;
if (ret) {
/* small size class but cache bin disabled. */
assert((uintptr_t)(*bin->stack_head) ==
cache_bin_preceding_junk);
}
return ret;
}
JEMALLOC_ALWAYS_INLINE bool
tcache_large_bin_disabled(szind_t ind, cache_bin_t *bin) {
assert(ind >= SC_NBINS);
assert(bin != NULL);
return (cache_bin_info_ncached_max(&bin->bin_info) == 0 ||
cache_bin_still_zero_initialized(bin));
return disabled;
}
JEMALLOC_ALWAYS_INLINE void *
@ -124,7 +102,8 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
if (unlikely(arena == NULL)) {
return NULL;
}
if (unlikely(tcache_small_bin_disabled(binind, bin))) {
if (unlikely(tcache_bin_disabled(binind, bin,
tcache->tcache_slow))) {
/* stats and zero are handled directly by the arena. */
return arena_malloc_hard(tsd_tsdn(tsd), arena, size,
binind, zero, /* slab */ true);
@ -157,8 +136,9 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
void *ret;
bool tcache_success;
assert(binind >= SC_NBINS && binind < tcache_nhbins_get(tcache));
cache_bin_t *bin = &tcache->bins[binind];
assert(binind >= SC_NBINS &&
!tcache_bin_disabled(binind, bin, tcache->tcache_slow));
ret = cache_bin_alloc(bin, &tcache_success);
assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) {
@ -180,7 +160,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
} else {
if (unlikely(zero)) {
size_t usize = sz_index2size(binind);
assert(usize <= tcache_max_get(tcache));
assert(usize <= tcache_max_get(tcache->tcache_slow));
memset(ret, 0, usize);
}
@ -214,12 +194,13 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
}
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
if (unlikely(tcache_small_bin_disabled(binind, bin))) {
if (unlikely(tcache_bin_disabled(binind, bin,
tcache->tcache_slow))) {
arena_dalloc_small(tsd_tsdn(tsd), ptr);
return;
}
cache_bin_sz_t max = cache_bin_info_ncached_max(
&bin->bin_info);
cache_bin_sz_t max = cache_bin_info_ncached_max_get(
bin, &bin->bin_info);
unsigned remain = max >> opt_lg_tcache_flush_small_div;
tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr);
@ -232,12 +213,13 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bool slow_path) {
assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SC_SMALL_MAXCLASS);
assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_max_get(tcache));
assert(tcache_salloc(tsd_tsdn(tsd), ptr) <=
tcache_max_get(tcache->tcache_slow));
cache_bin_t *bin = &tcache->bins[binind];
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
unsigned remain = cache_bin_info_ncached_max(
&bin->bin_info) >> opt_lg_tcache_flush_large_div;
unsigned remain = cache_bin_info_ncached_max_get(
bin, &bin->bin_info) >> opt_lg_tcache_flush_large_div;
tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret);

4
include/jemalloc/internal/tcache_structs.h
View File

@ -31,6 +31,8 @@ struct tcache_slow_s {
/* The arena this tcache is associated with. */
arena_t *arena;
/* The number of bins activated in the tcache. */
unsigned tcache_nbins;
/* Next bin to GC. */
szind_t next_gc_bin;
/* For small bins, fill (ncached_max >> lg_fill_div). */
@ -55,8 +57,6 @@ struct tcache_slow_s {
struct tcache_s {
tcache_slow_t *tcache_slow;
unsigned tcache_nhbins;
size_t tcache_max;
cache_bin_t bins[TCACHE_NBINS_MAX];
};

22
src/arena.c
View File

@ -163,17 +163,13 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) {
for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
cache_bin_t *cache_bin = &descriptor->bins[i];
if (cache_bin_disabled(cache_bin)) {
continue;
}
cache_bin_sz_t ncached, nstashed;
cache_bin_nitems_get_remote(cache_bin,
&cache_bin->bin_info, &ncached, &nstashed);
if ((i < SC_NBINS &&
tcache_small_bin_disabled(i, cache_bin)) ||
(i >= SC_NBINS &&
tcache_large_bin_disabled(i, cache_bin))) {
assert(ncached == 0 && nstashed == 0);
}
astats->tcache_bytes += ncached * sz_index2size(i);
astats->tcache_stashed_bytes += nstashed *
sz_index2size(i);
@ -730,11 +726,13 @@ arena_dalloc_promoted_impl(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
*/
safety_check_verify_redzone(ptr, usize, bumped_usize);
}
szind_t bumped_ind = sz_size2index(bumped_usize);
if (bumped_usize >= SC_LARGE_MINCLASS &&
tcache != NULL &&
bumped_usize <= tcache_max_get(tcache)) {
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
sz_size2index(bumped_usize), slow_path);
tcache != NULL && bumped_ind < TCACHE_NBINS_MAX &&
!tcache_bin_disabled(bumped_ind, &tcache->bins[bumped_ind],
tcache->tcache_slow)) {
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, bumped_ind,
slow_path);
} else {
large_dalloc(tsdn, edata);
}

20
src/cache_bin.c
View File

@ -5,10 +5,11 @@
#include "jemalloc/internal/cache_bin.h"
#include "jemalloc/internal/safety_check.h"
const uintptr_t disabled_bin = JUNK_ADDR;
void
cache_bin_info_init(cache_bin_info_t *info,
cache_bin_sz_t ncached_max) {
assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
size_t stack_size = (size_t)ncached_max * sizeof(void *);
assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
info->ncached_max = (cache_bin_sz_t)ncached_max;
@ -37,7 +38,6 @@ cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
*/
*size = sizeof(void *) * 2;
for (szind_t i = 0; i < ninfos; i++) {
assert(infos[i].ncached_max > 0);
*size += infos[i].ncached_max * sizeof(void *);
}
@ -98,13 +98,21 @@ cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc,
cache_bin_sz_t free_spots = cache_bin_diff(bin,
bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head);
assert(free_spots == bin_stack_size);
assert(cache_bin_ncached_get_local(bin, info) == 0);
if (!cache_bin_disabled(bin)) {
assert(cache_bin_ncached_get_local(bin, &bin->bin_info) == 0);
}
assert(cache_bin_empty_position_get(bin) == empty_position);
assert(bin_stack_size > 0 || empty_position == full_position);
}
bool
cache_bin_still_zero_initialized(cache_bin_t *bin) {
return bin->stack_head == NULL;
void
cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max) {
const void *fake_stack = cache_bin_disabled_bin_stack();
size_t fake_offset = 0;
cache_bin_info_t fake_info;
cache_bin_info_init(&fake_info, 0);
cache_bin_init(bin, &fake_info, (void *)fake_stack, &fake_offset);
cache_bin_info_init(&bin->bin_info, ncached_max);
assert(fake_offset == 0);
}

6
src/ctl.c
View File

@ -2317,7 +2317,7 @@ thread_tcache_max_ctl(tsd_t *tsd, const size_t *mib,
/* pointer to tcache_t always exists even with tcache disabled. */
tcache_t *tcache = tsd_tcachep_get(tsd);
assert(tcache != NULL);
oldval = tcache_max_get(tcache);
oldval = tcache_max_get(tcache->tcache_slow);
READ(oldval, size_t);
if (newp != NULL) {
@ -2332,7 +2332,7 @@ thread_tcache_max_ctl(tsd_t *tsd, const size_t *mib,
}
new_tcache_max = sz_s2u(new_tcache_max);
if(new_tcache_max != oldval) {
thread_tcache_max_and_nhbins_set(tsd, new_tcache_max);
thread_tcache_max_set(tsd, new_tcache_max);
}
}
@ -3155,7 +3155,7 @@ CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
CTL_RO_NL_GEN(arenas_tcache_max, global_do_not_change_tcache_maxclass, size_t)
CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned)
CTL_RO_NL_GEN(arenas_nhbins, global_do_not_change_nhbins, unsigned)
CTL_RO_NL_GEN(arenas_nhbins, global_do_not_change_nbins, unsigned)
CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t)
CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t)
CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t)

3
src/jemalloc.c
View File

@ -4140,7 +4140,8 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) {
tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind,
/* slow */ true, /* is_alloc */ true);
if (likely(tcache != NULL &&
ind < tcache_nhbins_get(tcache)) && progress < batch) {
!tcache_bin_disabled(ind, &tcache->bins[ind],
tcache->tcache_slow)) && progress < batch) {
if (bin == NULL) {
bin = &tcache->bins[ind];
}

240
src/tcache.c
View File

@ -60,10 +60,10 @@ unsigned opt_lg_tcache_flush_large_div = 1;
/*
* Number of cache bins enabled, including both large and small. This value
* is only used to initialize tcache_nhbins in the per-thread tcache.
* is only used to initialize tcache_nbins in the per-thread tcache.
* Directly modifying it will not affect threads already launched.
*/
unsigned global_do_not_change_nhbins;
unsigned global_do_not_change_nbins;
/*
* Max size class to be cached (can be small or large). This value is only used
* to initialize tcache_max in the per-thread tcache. Directly modifying it
@ -129,6 +129,7 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
assert(szind < SC_NBINS);
cache_bin_t *cache_bin = &tcache->bins[szind];
assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow));
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
@ -155,7 +156,7 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
* Reduce fill count by 2X. Limit lg_fill_div such that
* the fill count is always at least 1.
*/
if ((cache_bin_info_ncached_max(&cache_bin->bin_info)
if ((cache_bin_info_ncached_max_get(cache_bin, &cache_bin->bin_info)
>> (tcache_slow->lg_fill_div[szind] + 1)) >= 1) {
tcache_slow->lg_fill_div[szind]++;
}
@ -167,6 +168,7 @@ tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
/* Like the small GC; flush 3/4 of untouched items. */
assert(szind >= SC_NBINS);
cache_bin_t *cache_bin = &tcache->bins[szind];
assert(!tcache_bin_disabled(szind, cache_bin, tcache->tcache_slow));
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&cache_bin->bin_info);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
@ -187,8 +189,12 @@ tcache_event(tsd_t *tsd) {
bool is_small = (szind < SC_NBINS);
cache_bin_t *cache_bin = &tcache->bins[szind];
tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small);
if (tcache_bin_disabled(szind, cache_bin, tcache_slow)) {
goto label_done;
}
tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind,
is_small);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&cache_bin->bin_info);
if (low_water > 0) {
@ -210,8 +216,9 @@ tcache_event(tsd_t *tsd) {
}
cache_bin_low_water_set(cache_bin);
label_done:
tcache_slow->next_gc_bin++;
if (tcache_slow->next_gc_bin == tcache_nhbins_get(tcache)) {
if (tcache_slow->next_gc_bin == tcache_nbins_get(tcache_slow)) {
tcache_slow->next_gc_bin = 0;
}
}
@ -236,8 +243,9 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena,
void *ret;
assert(tcache_slow->arena != NULL);
unsigned nfill = cache_bin_info_ncached_max(&cache_bin->bin_info)
>> tcache_slow->lg_fill_div[binind];
assert(!tcache_bin_disabled(binind, cache_bin, tcache_slow));
unsigned nfill = cache_bin_info_ncached_max_get(cache_bin,
&cache_bin->bin_info) >> tcache_slow->lg_fill_div[binind];
arena_cache_bin_fill_small(tsdn, arena, cache_bin,
&cache_bin->bin_info, binind, nfill);
tcache_slow->bin_refilled[binind] = true;
@ -321,7 +329,7 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
if (small) {
assert(binind < SC_NBINS);
} else {
assert(binind < tcache_nhbins_get(tcache));
assert(binind < tcache_nbins_get(tcache_slow));
}
arena_t *tcache_arena = tcache_slow->arena;
assert(tcache_arena != NULL);
@ -508,6 +516,7 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
JEMALLOC_ALWAYS_INLINE void
tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
szind_t binind, unsigned rem, bool small) {
assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow));
tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small);
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
@ -551,6 +560,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
void
tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
szind_t binind, bool is_small) {
assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow));
cache_bin_info_t *info = &cache_bin->bin_info;
/*
* The two below are for assertion only. The content of original cached
@ -562,7 +572,8 @@ tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
info);
cache_bin_sz_t nstashed = cache_bin_nstashed_get_local(cache_bin, info);
assert(orig_cached + nstashed <= cache_bin_info_ncached_max(info));
assert(orig_cached + nstashed <=
cache_bin_info_ncached_max_get(cache_bin, info));
if (nstashed == 0) {
return;
}
@ -637,33 +648,11 @@ tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
}
static void
tcache_max_and_nhbins_init(tcache_t *tcache) {
assert(tcache != NULL);
tcache_default_settings_init(tcache_slow_t *tcache_slow) {
assert(tcache_slow != NULL);
assert(global_do_not_change_tcache_maxclass != 0);
assert(global_do_not_change_nhbins != 0);
tcache->tcache_max = global_do_not_change_tcache_maxclass;
tcache->tcache_nhbins = global_do_not_change_nhbins;
assert(tcache->tcache_nhbins == sz_size2index(tcache->tcache_max) + 1);
}
bool
tsd_tcache_enabled_data_init(tsd_t *tsd) {
/* Called upon tsd initialization. */
tsd_tcache_enabled_set(tsd, opt_tcache);
/*
* tcache is not available yet, but we need to set up its tcache_max
* and tcache_nhbins in advance.
*/
tcache_t *tcache = tsd_tcachep_get(tsd);
tcache_max_and_nhbins_init(tcache);
tsd_slow_update(tsd);
if (opt_tcache) {
/* Trigger tcache init. */
tsd_tcache_data_init(tsd, NULL);
}
return false;
assert(global_do_not_change_nbins != 0);
tcache_slow->tcache_nbins = global_do_not_change_nbins;
}
static void
@ -679,19 +668,15 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
/*
* We reserve cache bins for all small size classes, even if some may
* not get used (i.e. bins higher than tcache_nhbins). This allows
* not get used (i.e. bins higher than tcache_nbins). This allows
* the fast and common paths to access cache bin metadata safely w/o
* worrying about which ones are disabled.
*/
unsigned tcache_nhbins = tcache_nhbins_get(tcache);
unsigned n_reserved_bins = tcache_nhbins < SC_NBINS ? SC_NBINS
: tcache_nhbins;
memset(tcache->bins, 0, sizeof(cache_bin_t) * n_reserved_bins);
unsigned tcache_nbins = tcache_nbins_get(tcache_slow);
size_t cur_offset = 0;
cache_bin_preincrement(tcache_bin_info, tcache_nhbins, mem,
cache_bin_preincrement(tcache_bin_info, tcache_nbins, mem,
&cur_offset);
for (unsigned i = 0; i < tcache_nhbins; i++) {
for (unsigned i = 0; i < tcache_nbins; i++) {
if (i < SC_NBINS) {
tcache_slow->lg_fill_div[i] = 1;
tcache_slow->bin_refilled[i] = false;
@ -699,40 +684,40 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
= tcache_gc_item_delay_compute(i);
}
cache_bin_t *cache_bin = &tcache->bins[i];
if (tcache_bin_info[i].ncached_max > 0) {
cache_bin_init(cache_bin, &tcache_bin_info[i], mem,
&cur_offset);
} else {
cache_bin_init_disabled(cache_bin,
tcache_bin_info[i].ncached_max);
}
}
/*
* For small size classes beyond tcache_max(i.e.
* tcache_nhbins< NBINS), their cache bins are initialized to a state
* to safely and efficiently fail all fastpath alloc / free, so that
* no additional check around tcache_nhbins is needed on fastpath.
* Initialize all disabled bins to a state that can safely and
* efficiently fail all fastpath alloc / free, so that no additional
* check around tcache_nbins is needed on fastpath. Yet we still
* store the ncached_max in the bin_info for future usage.
*/
for (unsigned i = tcache_nhbins; i < SC_NBINS; i++) {
/* Disabled small bins. */
for (unsigned i = tcache_nbins; i < TCACHE_NBINS_MAX; i++) {
cache_bin_t *cache_bin = &tcache->bins[i];
void *fake_stack = mem;
size_t fake_offset = 0;
cache_bin_init(cache_bin, &tcache_bin_info[i], fake_stack,
&fake_offset);
assert(tcache_small_bin_disabled(i, cache_bin));
cache_bin_init_disabled(cache_bin,
tcache_bin_info[i].ncached_max);
assert(tcache_bin_disabled(i, cache_bin, tcache->tcache_slow));
}
cache_bin_postincrement(mem, &cur_offset);
if (config_debug) {
/* Sanity check that the whole stack is used. */
size_t size, alignment;
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins,
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins,
&size, &alignment);
assert(cur_offset == size);
}
}
static inline unsigned
tcache_ncached_max_compute(szind_t szind, unsigned current_nhbins) {
tcache_ncached_max_compute(szind_t szind) {
if (szind >= SC_NBINS) {
assert(szind < current_nhbins);
return opt_tcache_nslots_large;
}
unsigned slab_nregs = bin_infos[szind].nregs;
@ -788,32 +773,28 @@ tcache_ncached_max_compute(szind_t szind, unsigned current_nhbins) {
}
static void
tcache_bin_info_compute(cache_bin_info_t *tcache_bin_info,
unsigned tcache_nhbins) {
for (szind_t i = 0; i < tcache_nhbins; i++) {
unsigned ncached_max = tcache_ncached_max_compute(i,
tcache_nhbins);
tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
/*
* Compute the values for each bin, but for bins with indices larger
* than tcache_nbins, no items will be cached.
*/
for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
unsigned ncached_max = tcache_ncached_max_compute(i);
assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
cache_bin_info_init(&tcache_bin_info[i], ncached_max);
}
for (szind_t i = tcache_nhbins; i < SC_NBINS; i++) {
/* Disabled small bins. */
cache_bin_info_init(&tcache_bin_info[i], 0);
}
}
/* Initialize auto tcache (embedded in TSD). */
bool
tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) {
static bool
tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena,
cache_bin_info_t *tcache_bin_info) {
tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd);
tcache_t *tcache = tsd_tcachep_get_unsafe(tsd);
assert(cache_bin_still_zero_initialized(&tcache->bins[0]));
unsigned tcache_nhbins = tcache_nhbins_get(tcache);
unsigned tcache_nbins = tcache_nbins_get(tcache_slow);
size_t size, alignment;
/* Takes 146B stack space. */
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {0};
tcache_bin_info_compute(tcache_bin_info, tcache_nhbins);
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins,
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins,
&size, &alignment);
void *mem;
@ -860,6 +841,23 @@ tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) {
return false;
}
static bool
tsd_tcache_data_init_with_bin_settings(tsd_t *tsd, arena_t *arena,
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
assert(tcache_bin_info != NULL);
return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info);
}
/* Initialize auto tcache (embedded in TSD). */
static bool
tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) {
/* Takes 146B stack space. */
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}};
tcache_bin_info_compute(tcache_bin_info);
return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info);
}
/* Created manual tcache for tcache.create mallctl. */
tcache_t *
tcache_create_explicit(tsd_t *tsd) {
@ -868,11 +866,11 @@ tcache_create_explicit(tsd_t *tsd) {
* the beginning of the whole allocation (for freeing). The makes sure
* the cache bins have the requested alignment.
*/
unsigned tcache_nhbins = global_do_not_change_nhbins;
unsigned tcache_nbins = global_do_not_change_nbins;
size_t tcache_size, alignment;
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {0};
tcache_bin_info_compute(tcache_bin_info, tcache_nhbins);
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins,
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}};
tcache_bin_info_compute(tcache_bin_info);
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins,
&tcache_size, &alignment);
size_t size = tcache_size + sizeof(tcache_t)
@ -889,7 +887,7 @@ tcache_create_explicit(tsd_t *tsd) {
tcache_t *tcache = (void *)((byte_t *)mem + tcache_size);
tcache_slow_t *tcache_slow =
(void *)((byte_t *)mem + tcache_size + sizeof(tcache_t));
tcache_max_and_nhbins_init(tcache);
tcache_default_settings_init(tcache_slow);
tcache_init(tsd, tcache_slow, tcache, mem, tcache_bin_info);
tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache,
@ -898,13 +896,83 @@ tcache_create_explicit(tsd_t *tsd) {
return tcache;
}
bool
tsd_tcache_enabled_data_init(tsd_t *tsd) {
/* Called upon tsd initialization. */
tsd_tcache_enabled_set(tsd, opt_tcache);
/*
* tcache is not available yet, but we need to set up its tcache_nbins
* in advance.
*/
tcache_default_settings_init(tsd_tcache_slowp_get(tsd));
tsd_slow_update(tsd);
if (opt_tcache) {
/* Trigger tcache init. */
tsd_tcache_data_init(tsd, NULL);
}
return false;
}
void
tcache_enabled_set(tsd_t *tsd, bool enabled) {
bool was_enabled = tsd_tcache_enabled_get(tsd);
if (!was_enabled && enabled) {
tsd_tcache_data_init(tsd, NULL);
} else if (was_enabled && !enabled) {
tcache_cleanup(tsd);
}
/* Commit the state last. Above calls check current state. */
tsd_tcache_enabled_set(tsd, enabled);
tsd_slow_update(tsd);
}
void
thread_tcache_max_set(tsd_t *tsd, size_t tcache_max) {
assert(tcache_max <= TCACHE_MAXCLASS_LIMIT);
assert(tcache_max == sz_s2u(tcache_max));
tcache_t *tcache = tsd_tcachep_get(tsd);
tcache_slow_t *tcache_slow = tcache->tcache_slow;
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}};
assert(tcache != NULL && tcache_slow != NULL);
bool enabled = tcache_available(tsd);
arena_t *assigned_arena;
if (enabled) {
assigned_arena = tcache_slow->arena;
/* Carry over the bin settings during the reboot. */
tcache_bin_settings_backup(tcache, tcache_bin_info);
/* Shutdown and reboot the tcache for a clean slate. */
tcache_cleanup(tsd);
}
/*
* Still set tcache_nbins of the tcache even if the tcache is not
* available yet because the values are stored in tsd_t and are
* always available for changing.
*/
tcache_max_set(tcache_slow, tcache_max);
if (enabled) {
tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena,
tcache_bin_info);
}
assert(tcache_nbins_get(tcache_slow) == sz_size2index(tcache_max) + 1);
}
static void
tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
tcache_slow_t *tcache_slow = tcache->tcache_slow;
assert(tcache_slow->arena != NULL);
for (unsigned i = 0; i < tcache_nhbins_get(tcache); i++) {
for (unsigned i = 0; i < tcache_nbins_get(tcache_slow); i++) {
cache_bin_t *cache_bin = &tcache->bins[i];
if (tcache_bin_disabled(i, cache_bin, tcache_slow)) {
continue;
}
if (i < SC_NBINS) {
tcache_bin_flush_small(tsd, tcache, cache_bin, i, 0);
} else {
@ -974,8 +1042,7 @@ tcache_cleanup(tsd_t *tsd) {
tcache_destroy(tsd, tcache, true);
/* Make sure all bins used are reinitialized to the clean state. */
memset(tcache->bins, 0, sizeof(cache_bin_t) *
tcache_nhbins_get(tcache));
memset(tcache->bins, 0, sizeof(cache_bin_t) * TCACHE_NBINS_MAX);
}
void
@ -983,8 +1050,11 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
cassert(config_stats);
/* Merge and reset tcache stats. */
for (unsigned i = 0; i < tcache_nhbins_get(tcache); i++) {
for (unsigned i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) {
cache_bin_t *cache_bin = &tcache->bins[i];
if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) {
continue;
}
if (i < SC_NBINS) {
bin_t *bin = arena_bin_choose(tsdn, arena, i, NULL);
malloc_mutex_lock(tsdn, &bin->lock);
@ -1110,7 +1180,7 @@ bool
tcache_boot(tsdn_t *tsdn, base_t *base) {
global_do_not_change_tcache_maxclass = sz_s2u(opt_tcache_max);
assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT);
global_do_not_change_nhbins =
global_do_not_change_nbins =
sz_size2index(global_do_not_change_tcache_maxclass) + 1;
if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES,

132
test/unit/cache_bin.c
View File

@ -106,11 +106,13 @@ TEST_BEGIN(test_cache_bin) {
cache_bin_info_init(&info, ncached_max);
cache_bin_t bin;
test_bin_init(&bin, &info);
cache_bin_info_t *bin_info = &bin.bin_info;
/* Initialize to empty; should then have 0 elements. */
expect_d_eq(ncached_max, cache_bin_info_ncached_max(&info), "");
expect_true(cache_bin_ncached_get_local(&bin, &info) == 0, "");
expect_true(cache_bin_low_water_get(&bin, &info) == 0, "");
expect_d_eq(ncached_max, cache_bin_info_ncached_max_get(&bin,
&bin.bin_info), "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, "");
expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, "");
ptr = cache_bin_alloc_easy(&bin, &success);
expect_false(success, "Shouldn't successfully allocate when empty");
@ -127,14 +129,14 @@ TEST_BEGIN(test_cache_bin) {
void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0);
assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_ncached_get_local(&bin, &info) == i, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == i, "");
success = cache_bin_dalloc_easy(&bin, &ptrs[i]);
expect_true(success,
"Should be able to dalloc into a non-full cache bin.");
expect_true(cache_bin_low_water_get(&bin, &info) == 0,
expect_true(cache_bin_low_water_get(&bin, bin_info) == 0,
"Pushes and pops shouldn't change low water of zero.");
}
expect_true(cache_bin_ncached_get_local(&bin, &info) == ncached_max,
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max,
"");
success = cache_bin_dalloc_easy(&bin, &ptrs[ncached_max]);
expect_false(success, "Shouldn't be able to dalloc into a full bin.");
@ -142,9 +144,9 @@ TEST_BEGIN(test_cache_bin) {
cache_bin_low_water_set(&bin);
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_low_water_get(&bin, &info)
expect_true(cache_bin_low_water_get(&bin, bin_info)
== ncached_max - i, "");
expect_true(cache_bin_ncached_get_local(&bin, &info)
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
== ncached_max - i, "");
/*
* This should fail -- the easy variant can't change the low
@ -153,9 +155,9 @@ TEST_BEGIN(test_cache_bin) {
ptr = cache_bin_alloc_easy(&bin, &success);
expect_ptr_null(ptr, "");
expect_false(success, "");
expect_true(cache_bin_low_water_get(&bin, &info)
expect_true(cache_bin_low_water_get(&bin, bin_info)
== ncached_max - i, "");
expect_true(cache_bin_ncached_get_local(&bin, &info)
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
== ncached_max - i, "");
/* This should succeed, though. */
@ -163,13 +165,13 @@ TEST_BEGIN(test_cache_bin) {
expect_true(success, "");
expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1],
"Alloc should pop in stack order");
expect_true(cache_bin_low_water_get(&bin, &info)
expect_true(cache_bin_low_water_get(&bin, bin_info)
== ncached_max - i - 1, "");
expect_true(cache_bin_ncached_get_local(&bin, &info)
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
== ncached_max - i - 1, "");
}
/* Now we're empty -- all alloc attempts should fail. */
expect_true(cache_bin_ncached_get_local(&bin, &info) == 0, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, "");
ptr = cache_bin_alloc_easy(&bin, &success);
expect_ptr_null(ptr, "");
expect_false(success, "");
@ -185,7 +187,7 @@ TEST_BEGIN(test_cache_bin) {
for (cache_bin_sz_t i = ncached_max / 2; i < ncached_max; i++) {
cache_bin_dalloc_easy(&bin, &ptrs[i]);
}
expect_true(cache_bin_ncached_get_local(&bin, &info) == ncached_max,
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max,
"");
for (cache_bin_sz_t i = ncached_max - 1; i >= ncached_max / 2; i--) {
/*
@ -202,60 +204,64 @@ TEST_BEGIN(test_cache_bin) {
expect_ptr_null(ptr, "");
/* We're going to test filling -- we must be empty to start. */
while (cache_bin_ncached_get_local(&bin, &info)) {
while (cache_bin_ncached_get_local(&bin, bin_info)) {
cache_bin_alloc(&bin, &success);
expect_true(success, "");
}
/* Test fill. */
/* Try to fill all, succeed fully. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max, ncached_max);
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max,
ncached_max);
/* Try to fill all, succeed partially. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max,
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max,
ncached_max / 2);
/* Try to fill all, fail completely. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max, 0);
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, 0);
/* Try to fill some, succeed fully. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2,
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2,
ncached_max / 2);
/* Try to fill some, succeed partially. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2,
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2,
ncached_max / 4);
/* Try to fill some, fail completely. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2, 0);
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, 0);
do_flush_test(&bin, &info, ptrs, ncached_max, ncached_max);
do_flush_test(&bin, &info, ptrs, ncached_max, ncached_max / 2);
do_flush_test(&bin, &info, ptrs, ncached_max, 0);
do_flush_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_test(&bin, &info, ptrs, ncached_max / 2, 0);
do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max);
do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2);
do_flush_test(&bin, bin_info, ptrs, ncached_max, 0);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max * 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max / 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 1);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 0);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2,
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max,
ncached_max * 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max,
ncached_max / 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, ncached_max);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2,
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 4);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 1);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, &info, ptrs, 2, ncached_max);
do_batch_alloc_test(&bin, &info, ptrs, 2, 2);
do_batch_alloc_test(&bin, &info, ptrs, 2, 1);
do_batch_alloc_test(&bin, &info, ptrs, 2, 0);
do_batch_alloc_test(&bin, &info, ptrs, 1, 2);
do_batch_alloc_test(&bin, &info, ptrs, 1, 1);
do_batch_alloc_test(&bin, &info, ptrs, 1, 0);
do_batch_alloc_test(&bin, &info, ptrs, 0, 2);
do_batch_alloc_test(&bin, &info, ptrs, 0, 1);
do_batch_alloc_test(&bin, &info, ptrs, 0, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 0);
free(ptrs);
}
@ -328,6 +334,7 @@ TEST_BEGIN(test_cache_bin_stash) {
cache_bin_info_t info;
cache_bin_info_init(&info, ncached_max);
test_bin_init(&bin, &info);
cache_bin_info_t *bin_info = &bin.bin_info;
/*
* The content of this array is not accessed; instead the interior
@ -337,10 +344,10 @@ TEST_BEGIN(test_cache_bin_stash) {
assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
bool ret;
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_ncached_get_local(&bin, &info) ==
expect_true(cache_bin_ncached_get_local(&bin, bin_info) ==
(i / 2 + i % 2), "Wrong ncached value");
expect_true(cache_bin_nstashed_get_local(&bin, &info) == i / 2,
"Wrong nstashed value");
expect_true(cache_bin_nstashed_get_local(&bin, bin_info) ==
i / 2, "Wrong nstashed value");
if (i % 2 == 0) {
cache_bin_dalloc_easy(&bin, &ptrs[i]);
} else {
@ -362,18 +369,23 @@ TEST_BEGIN(test_cache_bin_stash) {
expect_true(diff % 2 == 0, "Should be able to alloc");
} else {
expect_false(ret, "Should not alloc stashed");
expect_true(cache_bin_nstashed_get_local(&bin, &info) ==
ncached_max / 2, "Wrong nstashed value");
expect_true(cache_bin_nstashed_get_local(&bin,
bin_info) == ncached_max / 2,
"Wrong nstashed value");
}
}
test_bin_init(&bin, &info);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max, 0);
do_flush_stashed_test(&bin, &info, ptrs, 0, ncached_max);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 4, ncached_max / 2);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 4, ncached_max / 4);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max, 0);
do_flush_stashed_test(&bin, bin_info, ptrs, 0, ncached_max);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 2);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4,
ncached_max / 2);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 4);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4,
ncached_max / 4);
}
TEST_END

44
test/unit/tcache_max.c
View File

@ -76,8 +76,11 @@ tcache_bytes_read_local(void) {
size_t tcache_bytes = 0;
tsd_t *tsd = tsd_fetch();
tcache_t *tcache = tcache_get(tsd);
for (szind_t i = 0; i < tcache_nhbins_get(tcache); i++) {
for (szind_t i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) {
cache_bin_t *cache_bin = &tcache->bins[i];
if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) {
continue;
}
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&cache_bin->bin_info);
tcache_bytes += ncached * sz_index2size(i);
@ -211,7 +214,7 @@ TEST_BEGIN(test_tcache_max) {
TEST_END
static size_t
tcache_max2nhbins(size_t tcache_max) {
tcache_max2nbins(size_t tcache_max) {
return sz_size2index(tcache_max) + 1;
}
@ -241,23 +244,24 @@ validate_tcache_stack(tcache_t *tcache) {
static void *
tcache_check(void *arg) {
size_t old_tcache_max, new_tcache_max, min_tcache_max, sz;
unsigned tcache_nhbins;
unsigned tcache_nbins;
tsd_t *tsd = tsd_fetch();
tcache_t *tcache = tsd_tcachep_get(tsd);
tcache_slow_t *tcache_slow = tcache->tcache_slow;
sz = sizeof(size_t);
new_tcache_max = *(size_t *)arg;
min_tcache_max = 1;
/*
* Check the default tcache_max and tcache_nhbins of each thread's
* Check the default tcache_max and tcache_nbins of each thread's
* auto tcache.
*/
old_tcache_max = tcache_max_get(tcache);
old_tcache_max = tcache_max_get(tcache_slow);
expect_zu_eq(old_tcache_max, opt_tcache_max,
"Unexpected default value for tcache_max");
tcache_nhbins = tcache_nhbins_get(tcache);
expect_zu_eq(tcache_nhbins, (size_t)global_do_not_change_nhbins,
"Unexpected default value for tcache_nhbins");
tcache_nbins = tcache_nbins_get(tcache_slow);
expect_zu_eq(tcache_nbins, (size_t)global_do_not_change_nbins,
"Unexpected default value for tcache_nbins");
validate_tcache_stack(tcache);
/*
@ -275,12 +279,12 @@ tcache_check(void *arg) {
assert_d_eq(mallctl("thread.tcache.max",
NULL, NULL, (void *)&temp_tcache_max, sz),.0,
"Unexpected.mallctl().failure");
old_tcache_max = tcache_max_get(tcache);
old_tcache_max = tcache_max_get(tcache_slow);
expect_zu_eq(old_tcache_max, TCACHE_MAXCLASS_LIMIT,
"Unexpected value for tcache_max");
tcache_nhbins = tcache_nhbins_get(tcache);
expect_zu_eq(tcache_nhbins, TCACHE_NBINS_MAX,
"Unexpected value for tcache_nhbins");
tcache_nbins = tcache_nbins_get(tcache_slow);
expect_zu_eq(tcache_nbins, TCACHE_NBINS_MAX,
"Unexpected value for tcache_nbins");
assert_d_eq(mallctl("thread.tcache.max",
(void *)&old_tcache_max, &sz,
(void *)&min_tcache_max, sz),.0,
@ -294,10 +298,10 @@ tcache_check(void *arg) {
(void *)&e0, bool_sz), 0, "Unexpected mallctl() error");
expect_false(e1, "Unexpected previous tcache state");
min_tcache_max = sz_s2u(min_tcache_max);
expect_zu_eq(tcache_max_get(tcache), min_tcache_max,
expect_zu_eq(tcache_max_get(tcache_slow), min_tcache_max,
"Unexpected value for tcache_max");
expect_zu_eq(tcache_nhbins_get(tcache),
tcache_max2nhbins(min_tcache_max), "Unexpected value for nhbins");
expect_zu_eq(tcache_nbins_get(tcache_slow),
tcache_max2nbins(min_tcache_max), "Unexpected value for nbins");
assert_d_eq(mallctl("thread.tcache.max",
(void *)&old_tcache_max, &sz,
(void *)&new_tcache_max, sz),.0,
@ -307,18 +311,18 @@ tcache_check(void *arg) {
validate_tcache_stack(tcache);
/*
* Check the thread's tcache_max and nhbins both through mallctl
* Check the thread's tcache_max and nbins both through mallctl
* and alloc tests.
*/
if (new_tcache_max > TCACHE_MAXCLASS_LIMIT) {
new_tcache_max = TCACHE_MAXCLASS_LIMIT;
}
old_tcache_max = tcache_max_get(tcache);
old_tcache_max = tcache_max_get(tcache_slow);
expect_zu_eq(old_tcache_max, new_tcache_max,
"Unexpected value for tcache_max");
tcache_nhbins = tcache_nhbins_get(tcache);
expect_zu_eq(tcache_nhbins, tcache_max2nhbins(new_tcache_max),
"Unexpected value for tcache_nhbins");
tcache_nbins = tcache_nbins_get(tcache_slow);
expect_zu_eq(tcache_nbins, tcache_max2nbins(new_tcache_max),
"Unexpected value for tcache_nbins");
for (unsigned alloc_option = alloc_option_start;
alloc_option < alloc_option_end;
alloc_option++) {