project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SEC: Expand option configurability.

Browse Source 
This change pulls the SEC options into a struct, which simplifies their handling
across various modules (e.g. PA needs to forward on SEC options from the
malloc_conf string, but it doesn't really need to know their names).  While
we're here, make some of the fixed constants configurable, and unify naming from
the configuration options to the internals.
This commit is contained in:
David Goldblatt
2021-01-26 18:35:18 -08:00
committed by David Goldblatt
parent ce9386370a
commit fb327368db
12 changed files with 185 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/arena.c
View File

@@ -1479,9 +1479,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
* so arena_hpa_global is not yet initialized.
*/
if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) {
if (pa_shard_enable_hpa(&arena->pa_shard,
&opt_hpa_opts, opt_hpa_sec_nshards, opt_hpa_sec_max_alloc,
opt_hpa_sec_max_bytes)) {
if (pa_shard_enable_hpa(&arena->pa_shard, &opt_hpa_opts,
&opt_hpa_sec_opts)) {
goto label_error;
}
}

27
src/ctl.c
View File

@@ -98,9 +98,11 @@ CTL_PROTO(opt_hpa_slab_max_alloc)
CTL_PROTO(opt_hpa_hugification_threshold)
CTL_PROTO(opt_hpa_dehugification_threshold)
CTL_PROTO(opt_hpa_dirty_mult)
CTL_PROTO(opt_hpa_sec_nshards)
CTL_PROTO(opt_hpa_sec_max_alloc)
CTL_PROTO(opt_hpa_sec_max_bytes)
CTL_PROTO(opt_hpa_sec_nshards)
CTL_PROTO(opt_hpa_sec_bytes_after_flush)
CTL_PROTO(opt_hpa_sec_batch_fill_extra)
CTL_PROTO(opt_metadata_thp)
CTL_PROTO(opt_retain)
CTL_PROTO(opt_dss)
@@ -406,9 +408,13 @@ static const ctl_named_node_t opt_node[] = {
{NAME("hpa_dehugification_threshold"),
CTL(opt_hpa_dehugification_threshold)},
{NAME("hpa_dirty_mult"), CTL(opt_hpa_dirty_mult)},
{NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
{NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
{NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
{NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
{NAME("hpa_sec_bytes_after_flush"),
CTL(opt_hpa_sec_bytes_after_flush)},
{NAME("hpa_sec_batch_fill_extra"),
CTL(opt_hpa_sec_batch_fill_extra)},
{NAME("metadata_thp"), CTL(opt_metadata_thp)},
{NAME("retain"), CTL(opt_retain)},
{NAME("dss"), CTL(opt_dss)},
@@ -2100,8 +2106,9 @@ CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool)
CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool)
CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool)
CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool)
/* HPA options. */
CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool)
CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_hugification_threshold,
opt_hpa_opts.hugification_threshold, size_t)
CTL_RO_NL_GEN(opt_hpa_dehugification_threshold,
@@ -2111,9 +2118,17 @@ CTL_RO_NL_GEN(opt_hpa_dehugification_threshold,
* its representation are internal implementation details.
*/
CTL_RO_NL_GEN(opt_hpa_dirty_mult, opt_hpa_opts.dirty_mult, fxp_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_max_bytes, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_nshards, size_t)
CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
/* HPA SEC options */
CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush,
size_t)
CTL_RO_NL_GEN(opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra,
size_t)
CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp],
const char *)
CTL_RO_NL_GEN(opt_retain, opt_retain, bool)

30
src/jemalloc.c
View File

@@ -153,11 +153,7 @@ malloc_mutex_t arenas_lock;
/* The global hpa, and whether it's on. */
bool opt_hpa = false;
hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT;
size_t opt_hpa_sec_max_alloc = 32 * 1024;
/* These settings correspond to a maximum of 1MB cached per arena. */
size_t opt_hpa_sec_max_bytes = 256 * 1024;
size_t opt_hpa_sec_nshards = 4;
sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT;
/*
* Arenas that are used to service external requests. Not all elements of the
@@ -1473,12 +1469,21 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
CONF_CONTINUE;
}
CONF_HANDLE_SIZE_T(opt_hpa_sec_max_alloc, "hpa_sec_max_alloc",
PAGE, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_max_bytes, "hpa_sec_max_bytes",
PAGE, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_nshards, "hpa_sec_nshards",
0, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards,
"hpa_sec_nshards", 0, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_alloc,
"hpa_sec_max_alloc", PAGE, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
"hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush,
"hpa_sec_bytes_after_flush", PAGE, 0,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
"hpa_sec_batch_fill_extra", PAGE, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
if (CONF_MATCH("slab_sizes")) {
if (CONF_MATCH_VALUE("default")) {
@@ -1777,8 +1782,7 @@ malloc_init_hard_a0_locked() {
}
} else if (opt_hpa) {
if (pa_shard_enable_hpa(&a0->pa_shard, &opt_hpa_opts,
opt_hpa_sec_nshards, opt_hpa_sec_max_alloc,
opt_hpa_sec_max_bytes)) {
&opt_hpa_sec_opts)) {
return true;
}
}

5
src/pa.c
View File

@@ -50,13 +50,12 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
bool
pa_shard_enable_hpa(pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts,
size_t sec_nshards, size_t sec_alloc_max, size_t sec_bytes_max) {
const sec_opts_t *hpa_sec_opts) {
if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base,
&shard->edata_cache, shard->ind, hpa_opts)) {
return true;
}
if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, sec_nshards,
sec_alloc_max, sec_bytes_max)) {
if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, hpa_sec_opts)) {
return true;
}
shard->ever_used_hpa = true;

63
src/sec.c
View File

@@ -19,12 +19,12 @@ sec_bin_init(sec_bin_t *bin) {
}
bool
sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t alloc_max,
size_t bytes_max) {
if (nshards > SEC_NSHARDS_MAX) {
nshards = SEC_NSHARDS_MAX;
sec_init(sec_t *sec, pai_t *fallback, const sec_opts_t *opts) {
size_t nshards_clipped = opts->nshards;
if (nshards_clipped > SEC_NSHARDS_MAX) {
nshards_clipped = SEC_NSHARDS_MAX;
}
for (size_t i = 0; i < nshards; i++) {
for (size_t i = 0; i < nshards_clipped; i++) {
sec_shard_t *shard = &sec->shards[i];
bool err = malloc_mutex_init(&shard->mtx, "sec_shard",
WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive);
@@ -39,15 +39,15 @@ sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t alloc_max,
shard->to_flush_next = 0;
}
sec->fallback = fallback;
sec->alloc_max = alloc_max;
if (sec->alloc_max > sz_pind2sz(SEC_NPSIZES - 1)) {
sec->alloc_max = sz_pind2sz(SEC_NPSIZES - 1);
size_t max_alloc_clipped = opts->max_alloc;
if (max_alloc_clipped > sz_pind2sz(SEC_NPSIZES - 1)) {
max_alloc_clipped = sz_pind2sz(SEC_NPSIZES - 1);
}
sec->bytes_max = bytes_max;
sec->bytes_after_flush = bytes_max / 2;
sec->batch_fill_extra = 4;
sec->nshards = nshards;
sec->opts = *opts;
sec->opts.nshards = nshards_clipped;
sec->opts.max_alloc = max_alloc_clipped;
/*
* Initialize these last so that an improper use of an SEC whose
@@ -83,8 +83,9 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
* when we multiply by the number of shards.
*/
uint64_t rand32 = prng_lg_range_u64(tsd_prng_statep_get(tsd), 32);
uint32_t idx = (uint32_t)((rand32 * (uint64_t)sec->nshards) >> 32);
assert(idx < (uint32_t)sec->nshards);
uint32_t idx =
(uint32_t)((rand32 * (uint64_t)sec->opts.nshards) >> 32);
assert(idx < (uint32_t)sec->opts.nshards);
*idxp = (uint8_t)idx;
}
return &sec->shards[*idxp];
@@ -99,7 +100,7 @@ sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
edata_list_active_t to_flush;
edata_list_active_init(&to_flush);
while (shard->bytes_cur > sec->bytes_after_flush) {
while (shard->bytes_cur > sec->opts.bytes_after_flush) {
/* Pick a victim. */
sec_bin_t *bin = &shard->bins[shard->to_flush_next];
@@ -155,7 +156,7 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
edata_list_active_t result;
edata_list_active_init(&result);
size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size,
1 + sec->batch_fill_extra, &result);
1 + sec->opts.batch_fill_extra, &result);
edata_t *ret = edata_list_active_first(&result);
if (ret != NULL) {
@@ -182,7 +183,7 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
bin->bytes_cur += new_cached_bytes;
shard->bytes_cur += new_cached_bytes;
if (shard->bytes_cur > sec->bytes_max) {
if (shard->bytes_cur > sec->opts.max_bytes) {
sec_flush_some_and_unlock(tsdn, sec, shard);
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
@@ -197,8 +198,8 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) {
sec_t *sec = (sec_t *)self;
if (zero || alignment > PAGE || sec->nshards == 0
|| size > sec->alloc_max) {
if (zero || alignment > PAGE || sec->opts.nshards == 0
|| size > sec->opts.max_alloc) {
return pai_alloc(tsdn, sec->fallback, size, alignment, zero);
}
pszind_t pszind = sz_psz2ind(size);
@@ -209,7 +210,8 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) {
malloc_mutex_lock(tsdn, &shard->mtx);
edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin);
if (edata == NULL) {
if (!bin->being_batch_filled && sec->batch_fill_extra > 0) {
if (!bin->being_batch_filled
&& sec->opts.batch_fill_extra > 0) {
bin->being_batch_filled = true;
do_batch_fill = true;
}
@@ -266,7 +268,7 @@ static void
sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
edata_t *edata) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
assert(shard->bytes_cur <= sec->bytes_max);
assert(shard->bytes_cur <= sec->opts.max_bytes);
size_t size = edata_size_get(edata);
pszind_t pszind = sz_psz2ind(size);
/*
@@ -277,7 +279,7 @@ sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
edata_list_active_prepend(&bin->freelist, edata);
bin->bytes_cur += size;
shard->bytes_cur += size;
if (shard->bytes_cur > sec->bytes_max) {
if (shard->bytes_cur > sec->opts.max_bytes) {
/*
* We've exceeded the shard limit. We make two nods in the
* direction of fragmentation avoidance: we flush everything in
@@ -297,7 +299,8 @@ sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
static void
sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
sec_t *sec = (sec_t *)self;
if (sec->nshards == 0 || edata_size_get(edata) > sec->alloc_max) {
if (sec->opts.nshards == 0
|| edata_size_get(edata) > sec->opts.max_alloc) {
pai_dalloc(tsdn, sec->fallback, edata);
return;
}
@@ -313,7 +316,7 @@ sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
void
sec_flush(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->nshards; i++) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
@@ -322,7 +325,7 @@ sec_flush(tsdn_t *tsdn, sec_t *sec) {
void
sec_disable(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->nshards; i++) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
sec->shards[i].enabled = false;
sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
@@ -333,7 +336,7 @@ sec_disable(tsdn_t *tsdn, sec_t *sec) {
void
sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
size_t sum = 0;
for (size_t i = 0; i < sec->nshards; i++) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
/*
* We could save these lock acquisitions by making bytes_cur
* atomic, but stats collection is rare anyways and we expect
@@ -349,7 +352,7 @@ sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
void
sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec,
mutex_prof_data_t *mutex_prof_data) {
for (size_t i = 0; i < sec->nshards; i++) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
malloc_mutex_prof_accum(tsdn, mutex_prof_data,
&sec->shards[i].mtx);
@@ -359,21 +362,21 @@ sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec,
void
sec_prefork2(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->nshards; i++) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_prefork(tsdn, &sec->shards[i].mtx);
}
}
void
sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->nshards; i++) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx);
}
}
void
sec_postfork_child(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->nshards; i++) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx);
}
}

4
src/stats.c
View File

@@ -1486,9 +1486,11 @@ stats_general_print(emitter_t *emitter) {
"opt.hpa_dirty_mult", emitter_type_string, &bufp);
}
}
OPT_WRITE_SIZE_T("hpa_sec_nshards")
OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
OPT_WRITE_SIZE_T("hpa_sec_nshards")
OPT_WRITE_SIZE_T("hpa_sec_bytes_after_flush")
OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra")
OPT_WRITE_CHAR_P("metadata_thp")
OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread")
OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms")