SEC: Allow arbitrarily many shards, cached sizes.

This commit is contained in:
David Goldblatt 2021-05-18 14:52:46 -07:00 committed by David Goldblatt
parent 11beab38bc
commit 36c6bfb963
7 changed files with 59 additions and 41 deletions

View File

@ -130,8 +130,8 @@ bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
* This isn't exposed to users; we allow late enablement of the HPA shard so * This isn't exposed to users; we allow late enablement of the HPA shard so
* that we can boot without worrying about the HPA, then turn it on in a0. * that we can boot without worrying about the HPA, then turn it on in a0.
*/ */
bool pa_shard_enable_hpa(pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts, bool pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
const sec_opts_t *hpa_sec_opts); const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts);
/* /*
* We stop using the HPA when custom extent hooks are installed, but still * We stop using the HPA when custom extent hooks are installed, but still
* redirect deallocations to it. * redirect deallocations to it.

View File

@ -13,20 +13,6 @@
* knowledge of the underlying PAI implementation). * knowledge of the underlying PAI implementation).
*/ */
/*
* This is a *small* extent cache, after all. Assuming 4k pages and an ngroup
* of 4, this allows caching of sizes up to 128k.
*/
#define SEC_NPSIZES 16
/*
* For now, we put a cap on the number of SECs an arena can have. There's no
* reason it can't be dynamic; it's just inconvenient. This number of shards
* are embedded in the arenas, so there's a space / configurability tradeoff
* here. Eventually, we should probably dynamically allocate only however many
* we require.
*/
#define SEC_NSHARDS_MAX 8
/* /*
* For now, this is just one field; eventually, we'll probably want to get more * For now, this is just one field; eventually, we'll probably want to get more
* fine-grained data out (like per-size class statistics). * fine-grained data out (like per-size class statistics).
@ -91,7 +77,7 @@ struct sec_shard_s {
* hooks are installed. * hooks are installed.
*/ */
bool enabled; bool enabled;
sec_bin_t bins[SEC_NPSIZES]; sec_bin_t *bins;
/* Number of bytes in all bins in the shard. */ /* Number of bytes in all bins in the shard. */
size_t bytes_cur; size_t bytes_cur;
/* The next pszind to flush in the flush-some pathways. */ /* The next pszind to flush in the flush-some pathways. */
@ -104,10 +90,12 @@ struct sec_s {
pai_t *fallback; pai_t *fallback;
sec_opts_t opts; sec_opts_t opts;
sec_shard_t shards[SEC_NSHARDS_MAX]; sec_shard_t *shards;
pszind_t npsizes;
}; };
bool sec_init(sec_t *sec, pai_t *fallback, const sec_opts_t *opts); bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
const sec_opts_t *opts);
void sec_flush(tsdn_t *tsdn, sec_t *sec); void sec_flush(tsdn_t *tsdn, sec_t *sec);
void sec_disable(tsdn_t *tsdn, sec_t *sec); void sec_disable(tsdn_t *tsdn, sec_t *sec);

View File

@ -1565,7 +1565,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
* so arena_hpa_global is not yet initialized. * so arena_hpa_global is not yet initialized.
*/ */
if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) { if (opt_hpa && ehooks_are_default(base_ehooks_get(base)) && ind != 0) {
if (pa_shard_enable_hpa(&arena->pa_shard, &opt_hpa_opts, if (pa_shard_enable_hpa(tsdn, &arena->pa_shard, &opt_hpa_opts,
&opt_hpa_sec_opts)) { &opt_hpa_sec_opts)) {
goto label_error; goto label_error;
} }

View File

@ -1781,7 +1781,7 @@ malloc_init_hard_a0_locked() {
opt_hpa = false; opt_hpa = false;
} }
} else if (opt_hpa) { } else if (opt_hpa) {
if (pa_shard_enable_hpa(&a0->pa_shard, &opt_hpa_opts, if (pa_shard_enable_hpa(TSDN_NULL, &a0->pa_shard, &opt_hpa_opts,
&opt_hpa_sec_opts)) { &opt_hpa_sec_opts)) {
return true; return true;
} }

View File

@ -49,13 +49,14 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
} }
bool bool
pa_shard_enable_hpa(pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts, pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
const sec_opts_t *hpa_sec_opts) { const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base, if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base,
&shard->edata_cache, shard->ind, hpa_opts)) { &shard->edata_cache, shard->ind, hpa_opts)) {
return true; return true;
} }
if (sec_init(&shard->hpa_sec, &shard->hpa_shard.pai, hpa_sec_opts)) { if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai,
hpa_sec_opts)) {
return true; return true;
} }
shard->ever_used_hpa = true; shard->ever_used_hpa = true;

View File

@ -19,35 +19,55 @@ sec_bin_init(sec_bin_t *bin) {
} }
bool bool
sec_init(sec_t *sec, pai_t *fallback, const sec_opts_t *opts) { sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
size_t nshards_clipped = opts->nshards; const sec_opts_t *opts) {
if (nshards_clipped > SEC_NSHARDS_MAX) { size_t max_alloc = opts->max_alloc & PAGE_MASK;
nshards_clipped = SEC_NSHARDS_MAX; pszind_t npsizes = sz_psz2ind(max_alloc);
if (sz_pind2sz(npsizes) > opts->max_alloc) {
npsizes--;
} }
for (size_t i = 0; i < nshards_clipped; i++) { size_t sz_shards = opts->nshards * sizeof(sec_shard_t);
sec_shard_t *shard = &sec->shards[i]; size_t sz_bins = opts->nshards * (size_t)npsizes * sizeof(sec_bin_t);
size_t sz_alloc = sz_shards + sz_bins;
void *dynalloc = base_alloc(tsdn, base, sz_alloc, CACHELINE);
if (dynalloc == NULL) {
return true;
}
sec_shard_t *shard_cur = (sec_shard_t *)dynalloc;
sec->shards = shard_cur;
sec_bin_t *bin_cur = (sec_bin_t *)&shard_cur[opts->nshards];
/* Just for asserts, below. */
sec_bin_t *bin_start = bin_cur;
for (size_t i = 0; i < opts->nshards; i++) {
sec_shard_t *shard = shard_cur;
shard_cur++;
bool err = malloc_mutex_init(&shard->mtx, "sec_shard", bool err = malloc_mutex_init(&shard->mtx, "sec_shard",
WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive); WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive);
if (err) { if (err) {
return true; return true;
} }
shard->enabled = true; shard->enabled = true;
for (pszind_t j = 0; j < SEC_NPSIZES; j++) { shard->bins = bin_cur;
for (pszind_t j = 0; j < npsizes; j++) {
sec_bin_init(&shard->bins[j]); sec_bin_init(&shard->bins[j]);
bin_cur++;
} }
shard->bytes_cur = 0; shard->bytes_cur = 0;
shard->to_flush_next = 0; shard->to_flush_next = 0;
} }
/*
* Should have exactly matched the bin_start to the first unused byte
* after the shards.
*/
assert((void *)shard_cur == (void *)bin_start);
/* And the last bin to use up the last bytes of the allocation. */
assert((char *)bin_cur == ((char *)dynalloc + sz_alloc));
sec->fallback = fallback; sec->fallback = fallback;
size_t max_alloc_clipped = opts->max_alloc;
if (max_alloc_clipped > sz_pind2sz(SEC_NPSIZES - 1)) {
max_alloc_clipped = sz_pind2sz(SEC_NPSIZES - 1);
}
sec->opts = *opts; sec->opts = *opts;
sec->opts.nshards = nshards_clipped; sec->npsizes = npsizes;
sec->opts.max_alloc = max_alloc_clipped;
/* /*
* Initialize these last so that an improper use of an SEC whose * Initialize these last so that an improper use of an SEC whose
@ -106,7 +126,7 @@ sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
/* Update our victim-picking state. */ /* Update our victim-picking state. */
shard->to_flush_next++; shard->to_flush_next++;
if (shard->to_flush_next == SEC_NPSIZES) { if (shard->to_flush_next == sec->npsizes) {
shard->to_flush_next = 0; shard->to_flush_next = 0;
} }
@ -249,7 +269,7 @@ sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
shard->bytes_cur = 0; shard->bytes_cur = 0;
edata_list_active_t to_flush; edata_list_active_t to_flush;
edata_list_active_init(&to_flush); edata_list_active_init(&to_flush);
for (pszind_t i = 0; i < SEC_NPSIZES; i++) { for (pszind_t i = 0; i < sec->npsizes; i++) {
sec_bin_t *bin = &shard->bins[i]; sec_bin_t *bin = &shard->bins[i];
bin->bytes_cur = 0; bin->bytes_cur = 0;
edata_list_active_concat(&to_flush, &bin->freelist); edata_list_active_concat(&to_flush, &bin->freelist);

View File

@ -37,7 +37,14 @@ test_sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t max_alloc,
opts.bytes_after_flush = max_bytes / 2; opts.bytes_after_flush = max_bytes / 2;
opts.batch_fill_extra = 4; opts.batch_fill_extra = 4;
bool err = sec_init(sec, fallback, &opts); /*
* We end up leaking this base, but that's fine; this test is
* short-running, and SECs are arena-scoped in reality.
*/
base_t *base = base_new(TSDN_NULL, /* ind */ 123,
&ehooks_default_extent_hooks);
bool err = sec_init(TSDN_NULL, sec, base, fallback, &opts);
assert_false(err, "Unexpected initialization failure"); assert_false(err, "Unexpected initialization failure");
} }
@ -412,10 +419,12 @@ TEST_BEGIN(test_nshards_0) {
sec_t sec; sec_t sec;
/* See the note above -- we can't use the real tsd. */ /* See the note above -- we can't use the real tsd. */
tsdn_t *tsdn = TSDN_NULL; tsdn_t *tsdn = TSDN_NULL;
base_t *base = base_new(TSDN_NULL, /* ind */ 123,
&ehooks_default_extent_hooks);
sec_opts_t opts = SEC_OPTS_DEFAULT; sec_opts_t opts = SEC_OPTS_DEFAULT;
opts.nshards = 0; opts.nshards = 0;
sec_init(&sec, &ta.pai, &opts); sec_init(TSDN_NULL, &sec, base, &ta.pai, &opts);
edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false); /* zero */ false);