project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor the tcache initiailization

Browse Source 
1. Pre-generate all default tcache ncached_max in tcache_boot;
2. Add getters returning default ncached_max and ncached_max_set;
3. Refactor tcache init so that it is always init with a given setting.
This commit is contained in:
guangli-dai 2023-10-17 20:17:42 -07:00 committed by Qi Wang
parent 8a22d10b83
commit 6fb3b6a8e4
7 changed files with 75 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/jemalloc/internal/cache_bin.h
View File

@ -719,8 +719,8 @@ void cache_bin_info_init(cache_bin_info_t *bin_info,
* Given an array of initialized cache_bin_info_ts, determine how big an
* allocation is required to initialize a full set of cache_bin_ts.
*/
void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
size_t *size, size_t *alignment);
void cache_bin_info_compute_alloc(const cache_bin_info_t *infos,
szind_t ninfos, size_t *size, size_t *alignment);
/*
* Actually initialize some cache bins. Callers should allocate the backing
@ -729,11 +729,11 @@ void cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
* cache_bin_postincrement. *alloc_cur will then point immediately past the end
* of the allocation.
*/
void cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos,
void cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos,
void *alloc, size_t *cur_offset);
void cache_bin_postincrement(void *alloc, size_t *cur_offset);
void cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc,
size_t *cur_offset);
void cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info,
void *alloc, size_t *cur_offset);
void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max);
bool cache_bin_stack_use_thp(void);

10
include/jemalloc/internal/tcache_externs.h
View File

@ -35,11 +35,6 @@ extern unsigned global_do_not_change_tcache_nbins;
*/
extern size_t global_do_not_change_tcache_maxclass;
/* Default bin info for each bin. */
extern cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX];
/* Records whether a bin's info is specified by malloc_conf. */
extern bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX];
/*
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
@ -60,9 +55,8 @@ void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, bool is_small);
bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur,
size_t len_left, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX],
bool bin_info_is_set[TCACHE_NBINS_MAX]);
bool tcache_bin_info_default_init(const char *bin_settings_segment_cur,
size_t len_left);
bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
bool tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size,
cache_bin_sz_t *ncached_max);

6
src/cache_bin.c
View File

@ -28,7 +28,7 @@ cache_bin_stack_use_thp(void) {
}
void
cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
cache_bin_info_compute_alloc(const cache_bin_info_t *infos, szind_t ninfos,
size_t *size, size_t *alignment) {
/* For the total bin stack region (per tcache), reserve 2 more slots so
* that
@ -51,7 +51,7 @@ cache_bin_info_compute_alloc(cache_bin_info_t *infos, szind_t ninfos,
}
void
cache_bin_preincrement(cache_bin_info_t *infos, szind_t ninfos, void *alloc,
cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos, void *alloc,
size_t *cur_offset) {
if (config_debug) {
size_t computed_size;
@ -76,7 +76,7 @@ cache_bin_postincrement(void *alloc, size_t *cur_offset) {
}
void
cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc,
cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
size_t *cur_offset) {
/*
* The full_position points to the lowest available space. Allocations

6
src/jemalloc.c
View File

@ -1323,14 +1323,12 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
CONF_CONTINUE;
}
if (CONF_MATCH("tcache_ncached_max")) {
bool err = tcache_bin_info_settings_parse(
v, vlen, opt_tcache_ncached_max,
opt_tcache_ncached_max_set);
bool err = tcache_bin_info_default_init(
v, vlen);
if (err) {
CONF_ERROR("Invalid settings for "
"tcache_ncached_max", k, klen, v,
vlen);
break;
}
CONF_CONTINUE;
}

82
src/tcache.c
View File

@ -71,14 +71,17 @@ unsigned global_do_not_change_tcache_nbins;
*/
size_t global_do_not_change_tcache_maxclass;
/* Default bin info for each bin. Will be initialized when thread starts. */
cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX] = {{0}};
/*
* Default bin info for each bin. Will be initialized in malloc_conf_init
* and tcache_boot and should not be modified after that.
*/
static cache_bin_info_t opt_tcache_ncached_max[TCACHE_NBINS_MAX] = {{0}};
/*
* Marks whether a bin's info is set already. This is used in
* tcache_bin_info_compute to avoid overwriting ncached_max specified by
* malloc_conf.
* malloc_conf. It should be set only when parsing malloc_conf.
*/
bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX] = {0};
static bool opt_tcache_ncached_max_set[TCACHE_NBINS_MAX] = {0};
tcaches_t *tcaches;
@ -599,6 +602,16 @@ tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
assert(head_content == *cache_bin->stack_head);
}
JET_EXTERN bool
tcache_get_default_ncached_max_set(szind_t ind) {
return opt_tcache_ncached_max_set[ind];
}
JET_EXTERN const cache_bin_info_t *
tcache_get_default_ncached_max(void) {
return opt_tcache_ncached_max;
}
bool
tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size,
cache_bin_sz_t *ncached_max) {
@ -687,7 +700,7 @@ tcache_default_settings_init(tcache_slow_t *tcache_slow) {
static void
tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
void *mem, cache_bin_info_t *tcache_bin_info) {
void *mem, const cache_bin_info_t *tcache_bin_info) {
tcache->tcache_slow = tcache_slow;
tcache_slow->tcache = tcache;
@ -809,8 +822,8 @@ tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
* than tcache_nbins, no items will be cached.
*/
for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
unsigned ncached_max = opt_tcache_ncached_max_set[i] ?
opt_tcache_ncached_max[i].ncached_max:
unsigned ncached_max = tcache_get_default_ncached_max_set(i) ?
(unsigned)tcache_get_default_ncached_max()[i].ncached_max:
tcache_ncached_max_compute(i);
assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
cache_bin_info_init(&tcache_bin_info[i], ncached_max);
@ -819,7 +832,7 @@ tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
static bool
tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena,
cache_bin_info_t *tcache_bin_info) {
const cache_bin_info_t *tcache_bin_info) {
tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd);
tcache_t *tcache = tsd_tcachep_get_unsafe(tsd);
@ -873,20 +886,11 @@ tsd_tcache_data_init_impl(tsd_t *tsd, arena_t *arena,
return false;
}
static bool
tsd_tcache_data_init_with_bin_settings(tsd_t *tsd, arena_t *arena,
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
assert(tcache_bin_info != NULL);
return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info);
}
/* Initialize auto tcache (embedded in TSD). */
static bool
tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) {
/* Takes 146B stack space. */
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}};
tcache_bin_info_compute(tcache_bin_info);
tsd_tcache_data_init(tsd_t *tsd, arena_t *arena,
const cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
assert(tcache_bin_info != NULL);
return tsd_tcache_data_init_impl(tsd, arena, tcache_bin_info);
}
@ -900,10 +904,8 @@ tcache_create_explicit(tsd_t *tsd) {
*/
unsigned tcache_nbins = global_do_not_change_tcache_nbins;
size_t tcache_size, alignment;
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}};
tcache_bin_info_compute(tcache_bin_info);
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nbins,
&tcache_size, &alignment);
cache_bin_info_compute_alloc(tcache_get_default_ncached_max(),
tcache_nbins, &tcache_size, &alignment);
size_t size = tcache_size + sizeof(tcache_t)
+ sizeof(tcache_slow_t);
@ -920,7 +922,8 @@ tcache_create_explicit(tsd_t *tsd) {
tcache_slow_t *tcache_slow =
(void *)((byte_t *)mem + tcache_size + sizeof(tcache_t));
tcache_default_settings_init(tcache_slow);
tcache_init(tsd, tcache_slow, tcache, mem, tcache_bin_info);
tcache_init(tsd, tcache_slow, tcache, mem,
tcache_get_default_ncached_max());
tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache,
arena_ichoose(tsd, NULL));
@ -941,7 +944,8 @@ tsd_tcache_enabled_data_init(tsd_t *tsd) {
if (opt_tcache) {
/* Trigger tcache init. */
tsd_tcache_data_init(tsd, NULL);
tsd_tcache_data_init(tsd, NULL,
tcache_get_default_ncached_max());
}
return false;
@ -952,7 +956,8 @@ tcache_enabled_set(tsd_t *tsd, bool enabled) {
bool was_enabled = tsd_tcache_enabled_get(tsd);
if (!was_enabled && enabled) {
tsd_tcache_data_init(tsd, NULL);
tsd_tcache_data_init(tsd, NULL,
tcache_get_default_ncached_max());
} else if (was_enabled && !enabled) {
tcache_cleanup(tsd);
}
@ -988,14 +993,14 @@ thread_tcache_max_set(tsd_t *tsd, size_t tcache_max) {
tcache_max_set(tcache_slow, tcache_max);
if (enabled) {
tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena,
tcache_bin_info);
tsd_tcache_data_init(tsd, assigned_arena, tcache_bin_info);
}
assert(tcache_nbins_get(tcache_slow) == sz_size2index(tcache_max) + 1);
}
bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur,
static bool
tcache_bin_info_settings_parse(const char *bin_settings_segment_cur,
size_t len_left, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX],
bool bin_info_is_set[TCACHE_NBINS_MAX]) {
do {
@ -1031,6 +1036,14 @@ bool tcache_bin_info_settings_parse(const char *bin_settings_segment_cur,
return false;
}
bool
tcache_bin_info_default_init(const char *bin_settings_segment_cur,
size_t len_left) {
return tcache_bin_info_settings_parse(bin_settings_segment_cur,
len_left, opt_tcache_ncached_max, opt_tcache_ncached_max_set);
}
bool
tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) {
assert(tcache_available(tsd));
@ -1047,7 +1060,7 @@ tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) {
arena_t *assigned_arena = tcache->tcache_slow->arena;
tcache_cleanup(tsd);
tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena,
tsd_tcache_data_init(tsd, assigned_arena,
tcache_bin_info);
return false;
@ -1272,6 +1285,13 @@ tcache_boot(tsdn_t *tsdn, base_t *base) {
assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT);
global_do_not_change_tcache_nbins =
sz_size2index(global_do_not_change_tcache_maxclass) + 1;
/*
* Pre-compute default bin info and store the results in
* opt_tcache_ncached_max. After the changes here,
* opt_tcache_ncached_max should not be modified and should always be
* accessed using tcache_get_default_ncached_max.
*/
tcache_bin_info_compute(opt_tcache_ncached_max);
if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES,
malloc_mutex_rank_exclusive)) {

20
test/unit/ncached_max.c
View File

@ -2,9 +2,11 @@
#include "test/san.h"
const char *malloc_conf =
"tcache_ncached_max:256-1024:1001|2048-2048:0,tcache_max:4096";
"tcache_ncached_max:256-1024:1001|2048-2048:0|8192-8192:1,tcache_max:4096";
extern void tcache_bin_info_compute(
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]);
extern bool tcache_get_default_ncached_max_set(szind_t ind);
extern const cache_bin_info_t *tcache_get_default_ncached_max(void);
static void
check_bins_info(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
@ -54,21 +56,23 @@ ncached_max_check(void* args) {
bool first_range = (i >= sz_size2index(256) &&
i <= sz_size2index(1024));
bool second_range = (i == sz_size2index(2048));
bool third_range = (i == sz_size2index(8192));
cache_bin_sz_t target_ncached_max = 0;
if (first_range || second_range) {
target_ncached_max = first_range ? 1001: 0;
expect_true(opt_tcache_ncached_max_set[i],
if (first_range || second_range || third_range) {
target_ncached_max = first_range ? 1001:
(second_range ? 0: 1);
expect_true(tcache_get_default_ncached_max_set(i),
"Unexpected state for bin %u", i);
expect_zu_eq(target_ncached_max,
tcache_bin_info[i].ncached_max,
"Unexpected generated ncached_max for bin %u", i);
expect_zu_eq(target_ncached_max,
tcache_get_default_ncached_max()[i].ncached_max,
"Unexpected pre-set ncached_max for bin %u", i);
} else {
expect_false(opt_tcache_ncached_max_set[i],
expect_false(tcache_get_default_ncached_max_set(i),
"Unexpected state for bin %u", i);
}
expect_zu_eq(target_ncached_max,
opt_tcache_ncached_max[i].ncached_max,
"Unexpected pre-set ncached_max for bin %u", i);
}
unsigned nbins = tcache_nbins_get(tcache_slow);
for (szind_t i = nbins; i < TCACHE_NBINS_MAX; i++) {

2
test/unit/tcache_max.c
View File

@ -2,8 +2,6 @@
#include "test/san.h"
const char *malloc_conf = TEST_SAN_UAF_ALIGN_DISABLE;
extern void tcache_bin_info_compute(
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]);
enum {
alloc_option_start = 0,