diff --git a/include/jemalloc/internal/cache_bin.h b/include/jemalloc/internal/cache_bin.h index e2da3b90..67565835 100644 --- a/include/jemalloc/internal/cache_bin.h +++ b/include/jemalloc/internal/cache_bin.h @@ -719,8 +719,8 @@ void cache_bin_info_init(cache_bin_info_t *bin_info, * Given an array of initialized cache_bin_info_ts, determine how big an * allocation is required to initialize a full set of cache_bin_ts. */ -void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, - size_t *size, size_t *alignment); +void cache_bin_info_compute_alloc(const cache_bin_info_t *infos, + szind_t ninfos, size_t *size, size_t *alignment); /* * Actually initialize some cache bins. Callers should allocate the backing @@ -729,11 +729,11 @@ void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, * cache_bin_postincrement. *alloc_cur will then point immediately past the end * of the allocation. */ -void cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, +void cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos, void *alloc, size_t *cur_offset); void cache_bin_postincrement(void *alloc, size_t *cur_offset); -void cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, - size_t *cur_offset); +void cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, + void *alloc, size_t *cur_offset); void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max); bool cache_bin_stack_use_thp(void); diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index 973dbfe9..732adacb 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -35,11 +35,6 @@ extern unsigned global_do_not_change_tcache_nbins; */ extern size_t global_do_not_change_tcache_maxclass; -/* Default bin info for each bin. */ -extern cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX]; -/* Records whether a bin's info is specified by malloc_conf. */ -extern bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX]; - /* * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and * usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are @@ -60,9 +55,8 @@ void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, unsigned rem); void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, bool is_small); -bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur, - size_t len_left, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX], - bool bin_info_is_set[TCACHE_NBINS_MAX]); +bool tcache_bin_info_default_init(const char *bin_settings_segment_cur, + size_t len_left); bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len); bool tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max); diff --git a/src/cache_bin.c b/src/cache_bin.c index 24dabd0b..c3b94e54 100644 --- a/src/cache_bin.c +++ b/src/cache_bin.c @@ -28,7 +28,7 @@ cache_bin_stack_use_thp(void) { } void -cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, +cache_bin_info_compute_alloc(const cache_bin_info_t *infos, szind_t ninfos, size_t *size, size_t *alignment) { /* For the total bin stack region (per tcache), reserve 2 more slots so * that @@ -51,7 +51,7 @@ cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, } void -cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc, +cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos, void *alloc, size_t *cur_offset) { if (config_debug) { size_t computed_size; @@ -76,7 +76,7 @@ cache_bin_postincrement(void *alloc, size_t *cur_offset) { } void -cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, +cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc, size_t *cur_offset) { /* * The full_position points to the lowest available space. Allocations diff --git a/src/jemalloc.c b/src/jemalloc.c index c77f2ef2..5da22a53 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1323,14 +1323,12 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_CONTINUE; } if (CONF_MATCH("tcache_ncached_max")) { - bool err = tcache_bin_info_settings_parse( - v, vlen, opt_tcache_ncached_max, - opt_tcache_ncached_max_set); + bool err = tcache_bin_info_default_init( + v, vlen); if (err) { CONF_ERROR("Invalid settings for " "tcache_ncached_max", k, klen, v, vlen); - break; } CONF_CONTINUE; } diff --git a/src/tcache.c b/src/tcache.c index a8eaf296..02627896 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -71,14 +71,17 @@ unsigned global_do_not_change_tcache_nbins; */ size_t global_do_not_change_tcache_maxclass; -/* Default bin info for each bin. Will be initialized when thread starts. */ -cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX] = {{0}}; +/* + * Default bin info for each bin. Will be initialized in malloc_conf_init + * and tcache_boot and should not be modified after that. + */ +static cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX] = {{0}}; /* * Marks whether a bin's info is set already. This is used in * tcache_bin_info_compute to avoid overwriting ncached_max specified by - * malloc_conf. + * malloc_conf. It should be set only when parsing malloc_conf. */ -bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX] = {0}; +static bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX] = {0}; tcaches_t *tcaches; @@ -599,6 +602,16 @@ tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, assert(head_content == *cache_bin->stack_head); } +JET_EXTERN bool +tcache_get_default_ncached_max_set(szind_t ind) { + return opt_tcache_ncached_max_set[ind]; +} + +JET_EXTERN const cache_bin_info_t * +tcache_get_default_ncached_max(void) { + return opt_tcache_ncached_max; +} + bool tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max) { @@ -687,7 +700,7 @@ tcache_default_settings_init(tcache_slow_t *tcache_slow) { static void tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, - void *mem, cache_bin_info_t *tcache_bin_info) { + void *mem, const cache_bin_info_t *tcache_bin_info) { tcache->tcache_slow = tcache_slow; tcache_slow->tcache = tcache; @@ -809,8 +822,8 @@ tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { * than tcache_nbins, no items will be cached. */ for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) { - unsigned ncached_max = opt_tcache_ncached_max_set[i] ? - opt_tcache_ncached_max[i].ncached_max: + unsigned ncached_max = tcache_get_default_ncached_max_set(i) ? + (unsigned)tcache_get_default_ncached_max()[i].ncached_max: tcache_ncached_max_compute(i); assert(ncached_max <= CACHE_BIN_NCACHED_MAX); cache_bin_info_init(&tcache_bin_info[i], ncached_max); @@ -819,7 +832,7 @@ tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { static bool tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena, - cache_bin_info_t *tcache_bin_info) { + const cache_bin_info_t *tcache_bin_info) { tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd); tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); @@ -873,20 +886,11 @@ tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena, return false; } -static bool -tsd_tcache_data_init_with_bin_settings(tsd_t *tsd, arena_t *arena, - cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { - assert(tcache_bin_info != NULL); - return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info); -} - /* Initialize auto tcache (embedded in TSD). */ static bool -tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) { - /* Takes 146B stack space. */ - cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}}; - tcache_bin_info_compute(tcache_bin_info); - +tsd_tcache_data_init(tsd_t *tsd, arena_t *arena, + const cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { + assert(tcache_bin_info != NULL); return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info); } @@ -900,10 +904,8 @@ tcache_create_explicit(tsd_t *tsd) { */ unsigned tcache_nbins = global_do_not_change_tcache_nbins; size_t tcache_size, alignment; - cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}}; - tcache_bin_info_compute(tcache_bin_info); - cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins, - &tcache_size, &alignment); + cache_bin_info_compute_alloc(tcache_get_default_ncached_max(), + tcache_nbins, &tcache_size, &alignment); size_t size = tcache_size + sizeof(tcache_t) + sizeof(tcache_slow_t); @@ -920,7 +922,8 @@ tcache_create_explicit(tsd_t *tsd) { tcache_slow_t *tcache_slow = (void *)((byte_t *)mem + tcache_size + sizeof(tcache_t)); tcache_default_settings_init(tcache_slow); - tcache_init(tsd, tcache_slow, tcache, mem, tcache_bin_info); + tcache_init(tsd, tcache_slow, tcache, mem, + tcache_get_default_ncached_max()); tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache, arena_ichoose(tsd, NULL)); @@ -941,7 +944,8 @@ tsd_tcache_enabled_data_init(tsd_t *tsd) { if (opt_tcache) { /* Trigger tcache init. */ - tsd_tcache_data_init(tsd, NULL); + tsd_tcache_data_init(tsd, NULL, + tcache_get_default_ncached_max()); } return false; @@ -952,7 +956,8 @@ tcache_enabled_set(tsd_t *tsd, bool enabled) { bool was_enabled = tsd_tcache_enabled_get(tsd); if (!was_enabled && enabled) { - tsd_tcache_data_init(tsd, NULL); + tsd_tcache_data_init(tsd, NULL, + tcache_get_default_ncached_max()); } else if (was_enabled && !enabled) { tcache_cleanup(tsd); } @@ -988,14 +993,14 @@ thread_tcache_max_set(tsd_t *tsd, size_t tcache_max) { tcache_max_set(tcache_slow, tcache_max); if (enabled) { - tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena, - tcache_bin_info); + tsd_tcache_data_init(tsd, assigned_arena, tcache_bin_info); } assert(tcache_nbins_get(tcache_slow) == sz_size2index(tcache_max) + 1); } -bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur, +static bool +tcache_bin_info_settings_parse(const char *bin_settings_segment_cur, size_t len_left, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX], bool bin_info_is_set[TCACHE_NBINS_MAX]) { do { @@ -1031,6 +1036,14 @@ bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur, return false; } +bool +tcache_bin_info_default_init(const char *bin_settings_segment_cur, + size_t len_left) { + return tcache_bin_info_settings_parse(bin_settings_segment_cur, + len_left, opt_tcache_ncached_max, opt_tcache_ncached_max_set); +} + + bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) { assert(tcache_available(tsd)); @@ -1047,7 +1060,7 @@ tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) { arena_t *assigned_arena = tcache->tcache_slow->arena; tcache_cleanup(tsd); - tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena, + tsd_tcache_data_init(tsd, assigned_arena, tcache_bin_info); return false; @@ -1272,6 +1285,13 @@ tcache_boot(tsdn_t *tsdn, base_t *base) { assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT); global_do_not_change_tcache_nbins = sz_size2index(global_do_not_change_tcache_maxclass) + 1; + /* + * Pre-compute default bin info and store the results in + * opt_tcache_ncached_max. After the changes here, + * opt_tcache_ncached_max should not be modified and should always be + * accessed using tcache_get_default_ncached_max. + */ + tcache_bin_info_compute(opt_tcache_ncached_max); if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES, malloc_mutex_rank_exclusive)) { diff --git a/test/unit/ncached_max.c b/test/unit/ncached_max.c index da35d7c9..1a0d2885 100644 --- a/test/unit/ncached_max.c +++ b/test/unit/ncached_max.c @@ -2,9 +2,11 @@ #include "test/san.h" const char *malloc_conf = -"tcache_ncached_max:256-1024:1001|2048-2048:0,tcache_max:4096"; +"tcache_ncached_max:256-1024:1001|2048-2048:0|8192-8192:1,tcache_max:4096"; extern void tcache_bin_info_compute( cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]); +extern bool tcache_get_default_ncached_max_set(szind_t ind); +extern const cache_bin_info_t *tcache_get_default_ncached_max(void); static void check_bins_info(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { @@ -54,21 +56,23 @@ ncached_max_check(void* args) { bool first_range = (i >= sz_size2index(256) && i <= sz_size2index(1024)); bool second_range = (i == sz_size2index(2048)); + bool third_range = (i == sz_size2index(8192)); cache_bin_sz_t target_ncached_max = 0; - if (first_range || second_range) { - target_ncached_max = first_range ? 1001: 0; - expect_true(opt_tcache_ncached_max_set[i], + if (first_range || second_range || third_range) { + target_ncached_max = first_range ? 1001: + (second_range ? 0: 1); + expect_true(tcache_get_default_ncached_max_set(i), "Unexpected state for bin %u", i); expect_zu_eq(target_ncached_max, tcache_bin_info[i].ncached_max, "Unexpected generated ncached_max for bin %u", i); + expect_zu_eq(target_ncached_max, + tcache_get_default_ncached_max()[i].ncached_max, + "Unexpected pre-set ncached_max for bin %u", i); } else { - expect_false(opt_tcache_ncached_max_set[i], + expect_false(tcache_get_default_ncached_max_set(i), "Unexpected state for bin %u", i); } - expect_zu_eq(target_ncached_max, - opt_tcache_ncached_max[i].ncached_max, - "Unexpected pre-set ncached_max for bin %u", i); } unsigned nbins = tcache_nbins_get(tcache_slow); for (szind_t i = nbins; i < TCACHE_NBINS_MAX; i++) { diff --git a/test/unit/tcache_max.c b/test/unit/tcache_max.c index 32eacadf..c740b5e7 100644 --- a/test/unit/tcache_max.c +++ b/test/unit/tcache_max.c @@ -2,8 +2,6 @@ #include "test/san.h" const char *malloc_conf = TEST_SAN_UAF_ALIGN_DISABLE; -extern void tcache_bin_info_compute( - cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]); enum { alloc_option_start = 0,