Refactor the tcache initiailization

1. Pre-generate all default tcache ncached_max in tcache_boot;
2. Add getters returning default ncached_max and ncached_max_set;
3. Refactor tcache init so that it is always init with a given setting.
This commit is contained in:
guangli-dai 2023-10-17 20:17:42 -07:00 committed by Qi Wang
parent 8a22d10b83
commit 6fb3b6a8e4
7 changed files with 75 additions and 61 deletions

View File

@ -719,8 +719,8 @@ void cache_bin_info_init(cache_bin_info_t *bin_info,
* Given an array of initialized cache_bin_info_ts, determine how big an * Given an array of initialized cache_bin_info_ts, determine how big an
* allocation is required to initialize a full set of cache_bin_ts. * allocation is required to initialize a full set of cache_bin_ts.
*/ */
void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, void cache_bin_info_compute_alloc(const cache_bin_info_t *infos,
size_t *size, size_t *alignment); szind_t ninfos, size_t *size, size_t *alignment);
/* /*
* Actually initialize some cache bins. Callers should allocate the backing * Actually initialize some cache bins. Callers should allocate the backing
@ -729,11 +729,11 @@ void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
* cache_bin_postincrement. *alloc_cur will then point immediately past the end * cache_bin_postincrement. *alloc_cur will then point immediately past the end
* of the allocation. * of the allocation.
*/ */
void cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos,
void *alloc, size_t *cur_offset); void *alloc, size_t *cur_offset);
void cache_bin_postincrement(void *alloc, size_t *cur_offset); void cache_bin_postincrement(void *alloc, size_t *cur_offset);
void cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, void cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info,
size_t *cur_offset); void *alloc, size_t *cur_offset);
void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max); void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max);
bool cache_bin_stack_use_thp(void); bool cache_bin_stack_use_thp(void);

View File

@ -35,11 +35,6 @@ extern unsigned global_do_not_change_tcache_nbins;
*/ */
extern size_t global_do_not_change_tcache_maxclass; extern size_t global_do_not_change_tcache_maxclass;
/* Default bin info for each bin. */
extern cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX];
/* Records whether a bin's info is specified by malloc_conf. */
extern bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX];
/* /*
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are * usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
@ -60,9 +55,8 @@ void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, unsigned rem); cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, bool is_small); cache_bin_t *cache_bin, szind_t binind, bool is_small);
bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur, bool tcache_bin_info_default_init(const char *bin_settings_segment_cur,
size_t len_left, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX], size_t len_left);
bool bin_info_is_set[TCACHE_NBINS_MAX]);
bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len); bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
bool tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size, bool tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size,
cache_bin_sz_t *ncached_max); cache_bin_sz_t *ncached_max);

View File

@ -28,7 +28,7 @@ cache_bin_stack_use_thp(void) {
} }
void void
cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos, cache_bin_info_compute_alloc(const cache_bin_info_t *infos, szind_t ninfos,
size_t *size, size_t *alignment) { size_t *size, size_t *alignment) {
/* For the total bin stack region (per tcache), reserve 2 more slots so /* For the total bin stack region (per tcache), reserve 2 more slots so
* that * that
@ -51,7 +51,7 @@ cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
} }
void void
cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc, cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos, void *alloc,
size_t *cur_offset) { size_t *cur_offset) {
if (config_debug) { if (config_debug) {
size_t computed_size; size_t computed_size;
@ -76,7 +76,7 @@ cache_bin_postincrement(void *alloc, size_t *cur_offset) {
} }
void void
cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc, cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
size_t *cur_offset) { size_t *cur_offset) {
/* /*
* The full_position points to the lowest available space. Allocations * The full_position points to the lowest available space. Allocations

View File

@ -1323,14 +1323,12 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
CONF_CONTINUE; CONF_CONTINUE;
} }
if (CONF_MATCH("tcache_ncached_max")) { if (CONF_MATCH("tcache_ncached_max")) {
bool err = tcache_bin_info_settings_parse( bool err = tcache_bin_info_default_init(
v, vlen, opt_tcache_ncached_max, v, vlen);
opt_tcache_ncached_max_set);
if (err) { if (err) {
CONF_ERROR("Invalid settings for " CONF_ERROR("Invalid settings for "
"tcache_ncached_max", k, klen, v, "tcache_ncached_max", k, klen, v,
vlen); vlen);
break;
} }
CONF_CONTINUE; CONF_CONTINUE;
} }

View File

@ -71,14 +71,17 @@ unsigned global_do_not_change_tcache_nbins;
*/ */
size_t global_do_not_change_tcache_maxclass; size_t global_do_not_change_tcache_maxclass;
/* Default bin info for each bin. Will be initialized when thread starts. */ /*
cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX] = {{0}}; * Default bin info for each bin. Will be initialized in malloc_conf_init
* and tcache_boot and should not be modified after that.
*/
static cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX] = {{0}};
/* /*
* Marks whether a bin's info is set already. This is used in * Marks whether a bin's info is set already. This is used in
* tcache_bin_info_compute to avoid overwriting ncached_max specified by * tcache_bin_info_compute to avoid overwriting ncached_max specified by
* malloc_conf. * malloc_conf. It should be set only when parsing malloc_conf.
*/ */
bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX] = {0}; static bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX] = {0};
tcaches_t *tcaches; tcaches_t *tcaches;
@ -599,6 +602,16 @@ tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
assert(head_content == *cache_bin->stack_head); assert(head_content == *cache_bin->stack_head);
} }
JET_EXTERN bool
tcache_get_default_ncached_max_set(szind_t ind) {
return opt_tcache_ncached_max_set[ind];
}
JET_EXTERN const cache_bin_info_t *
tcache_get_default_ncached_max(void) {
return opt_tcache_ncached_max;
}
bool bool
tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size, tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size,
cache_bin_sz_t *ncached_max) { cache_bin_sz_t *ncached_max) {
@ -687,7 +700,7 @@ tcache_default_settings_init(tcache_slow_t *tcache_slow) {
static void static void
tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
void *mem, cache_bin_info_t *tcache_bin_info) { void *mem, const cache_bin_info_t *tcache_bin_info) {
tcache->tcache_slow = tcache_slow; tcache->tcache_slow = tcache_slow;
tcache_slow->tcache = tcache; tcache_slow->tcache = tcache;
@ -809,8 +822,8 @@ tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
* than tcache_nbins, no items will be cached. * than tcache_nbins, no items will be cached.
*/ */
for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) { for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
unsigned ncached_max = opt_tcache_ncached_max_set[i] ? unsigned ncached_max = tcache_get_default_ncached_max_set(i) ?
opt_tcache_ncached_max[i].ncached_max: (unsigned)tcache_get_default_ncached_max()[i].ncached_max:
tcache_ncached_max_compute(i); tcache_ncached_max_compute(i);
assert(ncached_max <= CACHE_BIN_NCACHED_MAX); assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
cache_bin_info_init(&tcache_bin_info[i], ncached_max); cache_bin_info_init(&tcache_bin_info[i], ncached_max);
@ -819,7 +832,7 @@ tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
static bool static bool
tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena, tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena,
cache_bin_info_t *tcache_bin_info) { const cache_bin_info_t *tcache_bin_info) {
tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd); tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd);
tcache_t *tcache = tsd_tcachep_get_unsafe(tsd); tcache_t *tcache = tsd_tcachep_get_unsafe(tsd);
@ -873,20 +886,11 @@ tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena,
return false; return false;
} }
static bool
tsd_tcache_data_init_with_bin_settings(tsd_t *tsd, arena_t *arena,
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
assert(tcache_bin_info != NULL);
return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info);
}
/* Initialize auto tcache (embedded in TSD). */ /* Initialize auto tcache (embedded in TSD). */
static bool static bool
tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) { tsd_tcache_data_init(tsd_t *tsd, arena_t *arena,
/* Takes 146B stack space. */ const cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}}; assert(tcache_bin_info != NULL);
tcache_bin_info_compute(tcache_bin_info);
return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info); return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info);
} }
@ -900,10 +904,8 @@ tcache_create_explicit(tsd_t *tsd) {
*/ */
unsigned tcache_nbins = global_do_not_change_tcache_nbins; unsigned tcache_nbins = global_do_not_change_tcache_nbins;
size_t tcache_size, alignment; size_t tcache_size, alignment;
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}}; cache_bin_info_compute_alloc(tcache_get_default_ncached_max(),
tcache_bin_info_compute(tcache_bin_info); tcache_nbins, &tcache_size, &alignment);
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins,
&tcache_size, &alignment);
size_t size = tcache_size + sizeof(tcache_t) size_t size = tcache_size + sizeof(tcache_t)
+ sizeof(tcache_slow_t); + sizeof(tcache_slow_t);
@ -920,7 +922,8 @@ tcache_create_explicit(tsd_t *tsd) {
tcache_slow_t *tcache_slow = tcache_slow_t *tcache_slow =
(void *)((byte_t *)mem + tcache_size + sizeof(tcache_t)); (void *)((byte_t *)mem + tcache_size + sizeof(tcache_t));
tcache_default_settings_init(tcache_slow); tcache_default_settings_init(tcache_slow);
tcache_init(tsd, tcache_slow, tcache, mem, tcache_bin_info); tcache_init(tsd, tcache_slow, tcache, mem,
tcache_get_default_ncached_max());
tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache, tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache,
arena_ichoose(tsd, NULL)); arena_ichoose(tsd, NULL));
@ -941,7 +944,8 @@ tsd_tcache_enabled_data_init(tsd_t *tsd) {
if (opt_tcache) { if (opt_tcache) {
/* Trigger tcache init. */ /* Trigger tcache init. */
tsd_tcache_data_init(tsd, NULL); tsd_tcache_data_init(tsd, NULL,
tcache_get_default_ncached_max());
} }
return false; return false;
@ -952,7 +956,8 @@ tcache_enabled_set(tsd_t *tsd, bool enabled) {
bool was_enabled = tsd_tcache_enabled_get(tsd); bool was_enabled = tsd_tcache_enabled_get(tsd);
if (!was_enabled && enabled) { if (!was_enabled && enabled) {
tsd_tcache_data_init(tsd, NULL); tsd_tcache_data_init(tsd, NULL,
tcache_get_default_ncached_max());
} else if (was_enabled && !enabled) { } else if (was_enabled && !enabled) {
tcache_cleanup(tsd); tcache_cleanup(tsd);
} }
@ -988,14 +993,14 @@ thread_tcache_max_set(tsd_t *tsd, size_t tcache_max) {
tcache_max_set(tcache_slow, tcache_max); tcache_max_set(tcache_slow, tcache_max);
if (enabled) { if (enabled) {
tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena, tsd_tcache_data_init(tsd, assigned_arena, tcache_bin_info);
tcache_bin_info);
} }
assert(tcache_nbins_get(tcache_slow) == sz_size2index(tcache_max) + 1); assert(tcache_nbins_get(tcache_slow) == sz_size2index(tcache_max) + 1);
} }
bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur, static bool
tcache_bin_info_settings_parse(const char *bin_settings_segment_cur,
size_t len_left, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX], size_t len_left, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX],
bool bin_info_is_set[TCACHE_NBINS_MAX]) { bool bin_info_is_set[TCACHE_NBINS_MAX]) {
do { do {
@ -1031,6 +1036,14 @@ bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur,
return false; return false;
} }
bool
tcache_bin_info_default_init(const char *bin_settings_segment_cur,
size_t len_left) {
return tcache_bin_info_settings_parse(bin_settings_segment_cur,
len_left, opt_tcache_ncached_max, opt_tcache_ncached_max_set);
}
bool bool
tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) { tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) {
assert(tcache_available(tsd)); assert(tcache_available(tsd));
@ -1047,7 +1060,7 @@ tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) {
arena_t *assigned_arena = tcache->tcache_slow->arena; arena_t *assigned_arena = tcache->tcache_slow->arena;
tcache_cleanup(tsd); tcache_cleanup(tsd);
tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena, tsd_tcache_data_init(tsd, assigned_arena,
tcache_bin_info); tcache_bin_info);
return false; return false;
@ -1272,6 +1285,13 @@ tcache_boot(tsdn_t *tsdn, base_t *base) {
assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT); assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT);
global_do_not_change_tcache_nbins = global_do_not_change_tcache_nbins =
sz_size2index(global_do_not_change_tcache_maxclass) + 1; sz_size2index(global_do_not_change_tcache_maxclass) + 1;
/*
* Pre-compute default bin info and store the results in
* opt_tcache_ncached_max. After the changes here,
* opt_tcache_ncached_max should not be modified and should always be
* accessed using tcache_get_default_ncached_max.
*/
tcache_bin_info_compute(opt_tcache_ncached_max);
if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES, if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES,
malloc_mutex_rank_exclusive)) { malloc_mutex_rank_exclusive)) {

View File

@ -2,9 +2,11 @@
#include "test/san.h" #include "test/san.h"
const char *malloc_conf = const char *malloc_conf =
"tcache_ncached_max:256-1024:1001|2048-2048:0,tcache_max:4096"; "tcache_ncached_max:256-1024:1001|2048-2048:0|8192-8192:1,tcache_max:4096";
extern void tcache_bin_info_compute( extern void tcache_bin_info_compute(
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]); cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]);
extern bool tcache_get_default_ncached_max_set(szind_t ind);
extern const cache_bin_info_t *tcache_get_default_ncached_max(void);
static void static void
check_bins_info(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { check_bins_info(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
@ -54,21 +56,23 @@ ncached_max_check(void* args) {
bool first_range = (i >= sz_size2index(256) && bool first_range = (i >= sz_size2index(256) &&
i <= sz_size2index(1024)); i <= sz_size2index(1024));
bool second_range = (i == sz_size2index(2048)); bool second_range = (i == sz_size2index(2048));
bool third_range = (i == sz_size2index(8192));
cache_bin_sz_t target_ncached_max = 0; cache_bin_sz_t target_ncached_max = 0;
if (first_range || second_range) { if (first_range || second_range || third_range) {
target_ncached_max = first_range ? 1001: 0; target_ncached_max = first_range ? 1001:
expect_true(opt_tcache_ncached_max_set[i], (second_range ? 0: 1);
expect_true(tcache_get_default_ncached_max_set(i),
"Unexpected state for bin %u", i); "Unexpected state for bin %u", i);
expect_zu_eq(target_ncached_max, expect_zu_eq(target_ncached_max,
tcache_bin_info[i].ncached_max, tcache_bin_info[i].ncached_max,
"Unexpected generated ncached_max for bin %u", i); "Unexpected generated ncached_max for bin %u", i);
expect_zu_eq(target_ncached_max,
tcache_get_default_ncached_max()[i].ncached_max,
"Unexpected pre-set ncached_max for bin %u", i);
} else { } else {
expect_false(opt_tcache_ncached_max_set[i], expect_false(tcache_get_default_ncached_max_set(i),
"Unexpected state for bin %u", i); "Unexpected state for bin %u", i);
} }
expect_zu_eq(target_ncached_max,
opt_tcache_ncached_max[i].ncached_max,
"Unexpected pre-set ncached_max for bin %u", i);
} }
unsigned nbins = tcache_nbins_get(tcache_slow); unsigned nbins = tcache_nbins_get(tcache_slow);
for (szind_t i = nbins; i < TCACHE_NBINS_MAX; i++) { for (szind_t i = nbins; i < TCACHE_NBINS_MAX; i++) {

View File

@ -2,8 +2,6 @@
#include "test/san.h" #include "test/san.h"
const char *malloc_conf = TEST_SAN_UAF_ALIGN_DISABLE; const char *malloc_conf = TEST_SAN_UAF_ALIGN_DISABLE;
extern void tcache_bin_info_compute(
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]);
enum { enum {
alloc_option_start = 0, alloc_option_start = 0,