diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index b3fc8e2b..f6d0a7c3 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -66,12 +66,7 @@ struct pa_shard_s { */ atomic_zu_t nactive; - /* - * An interface for page allocation from the ecache framework (i.e. a - * cascade of ecache_dirty, ecache_muzzy, ecache_retained). Right now - * this is the *only* pai, but we'll soon grow another. - */ - pai_t ecache_pai; + /* Allocates from a PAC. */ pac_t pac; /* The source of edata_t objects. */ diff --git a/include/jemalloc/internal/pac.h b/include/jemalloc/internal/pac.h index 302ac078..2d02bda0 100644 --- a/include/jemalloc/internal/pac.h +++ b/include/jemalloc/internal/pac.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_PAC_H #define JEMALLOC_INTERNAL_PAC_H +#include "jemalloc/internal/pai.h" + /* * Page allocator classic; an implementation of the PAI interface that: * - Can be used for arenas with custom extent hooks. @@ -71,6 +73,11 @@ struct pac_stats_s { typedef struct pac_s pac_t; struct pac_s { + /* + * Must be the first member (we convert it to a PAC given only a + * pointer). The handle to the allocation interface. + */ + pai_t pai; /* * Collections of extents that were previously allocated. These are * used when allocating extents, in an attempt to re-use address space. diff --git a/src/pa.c b/src/pa.c index 6a3db3c6..f068fd96 100644 --- a/src/pa.c +++ b/src/pa.c @@ -1,14 +1,6 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" -static edata_t *ecache_pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero); -static bool ecache_pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool zero); -static bool ecache_pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size); -static void ecache_pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata); - static void pa_nactive_add(pa_shard_t *shard, size_t add_pages) { atomic_fetch_add_zu(&shard->nactive, add_pages, ATOMIC_RELAXED); @@ -44,11 +36,6 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base, shard->emap = emap; shard->base = base; - shard->ecache_pai.alloc = &ecache_pai_alloc; - shard->ecache_pai.expand = &ecache_pai_expand; - shard->ecache_pai.shrink = &ecache_pai_shrink; - shard->ecache_pai.dalloc = &ecache_pai_dalloc; - return false; } @@ -62,43 +49,13 @@ pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) { pac_destroy(tsdn, &shard->pac); } -static inline bool -pa_shard_may_have_muzzy(pa_shard_t *shard) { - return pac_decay_ms_get(&shard->pac, extent_state_muzzy) != 0; -} - -static edata_t * -ecache_pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, - bool zero) { - pa_shard_t *shard = - (pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai)); - - ehooks_t *ehooks = pa_shard_ehooks_get(shard); - edata_t *edata = ecache_alloc(tsdn, &shard->pac, ehooks, - &shard->pac.ecache_dirty, NULL, size, alignment, zero); - - if (edata == NULL && pa_shard_may_have_muzzy(shard)) { - edata = ecache_alloc(tsdn, &shard->pac, ehooks, - &shard->pac.ecache_muzzy, NULL, size, alignment, zero); - } - if (edata == NULL) { - edata = ecache_alloc_grow(tsdn, &shard->pac, ehooks, - &shard->pac.ecache_retained, NULL, size, alignment, zero); - if (config_stats && edata != NULL) { - atomic_fetch_add_zu(&shard->pac.stats->pac_mapped, size, - ATOMIC_RELAXED); - } - } - return edata; -} - edata_t * pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, bool slab, szind_t szind, bool zero) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - edata_t *edata = pai_alloc(tsdn, &shard->ecache_pai, size, alignment, + edata_t *edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero); if (edata != NULL) { @@ -113,48 +70,6 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, return edata; } -static bool -ecache_pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, - size_t new_size, bool zero) { - pa_shard_t *shard = - (pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai)); - - ehooks_t *ehooks = pa_shard_ehooks_get(shard); - void *trail_begin = edata_past_get(edata); - - size_t mapped_add = 0; - size_t expand_amount = new_size - old_size; - - if (ehooks_merge_will_fail(ehooks)) { - return true; - } - edata_t *trail = ecache_alloc(tsdn, &shard->pac, ehooks, - &shard->pac.ecache_dirty, trail_begin, expand_amount, PAGE, zero); - if (trail == NULL) { - trail = ecache_alloc(tsdn, &shard->pac, ehooks, - &shard->pac.ecache_muzzy, trail_begin, expand_amount, PAGE, - zero); - } - if (trail == NULL) { - trail = ecache_alloc_grow(tsdn, &shard->pac, ehooks, - &shard->pac.ecache_retained, trail_begin, expand_amount, - PAGE, zero); - mapped_add = expand_amount; - } - if (trail == NULL) { - return true; - } - if (extent_merge_wrapper(tsdn, &shard->pac, ehooks, edata, trail)) { - extent_dalloc_wrapper(tsdn, &shard->pac, ehooks, trail); - return true; - } - if (config_stats && mapped_add > 0) { - atomic_fetch_add_zu(&shard->pac.stats->pac_mapped, mapped_add, - ATOMIC_RELAXED); - } - return false; -} - bool pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, size_t new_size, szind_t szind, bool zero) { @@ -164,7 +79,7 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, size_t expand_amount = new_size - old_size; - bool error = pai_expand(tsdn, &shard->ecache_pai, edata, old_size, + bool error = pai_expand(tsdn, &shard->pac.pai, edata, old_size, new_size, zero); if (error) { return true; @@ -176,30 +91,6 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, return false; } -static bool -ecache_pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, - size_t new_size) { - pa_shard_t *shard = - (pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai)); - - ehooks_t *ehooks = pa_shard_ehooks_get(shard); - size_t shrink_amount = old_size - new_size; - - - if (ehooks_split_will_fail(ehooks)) { - return true; - } - - edata_t *trail = extent_split_wrapper(tsdn, &shard->pac, ehooks, edata, - new_size, shrink_amount); - if (trail == NULL) { - return true; - } - ecache_dalloc(tsdn, &shard->pac, ehooks, &shard->pac.ecache_dirty, - trail); - return false; -} - bool pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, size_t new_size, szind_t szind, bool *generated_dirty) { @@ -209,7 +100,7 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, size_t shrink_amount = old_size - new_size; *generated_dirty = false; - bool error = pai_shrink(tsdn, &shard->ecache_pai, edata, old_size, + bool error = pai_shrink(tsdn, &shard->pac.pai, edata, old_size, new_size); if (error) { return true; @@ -222,15 +113,6 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, return false; } -static void -ecache_pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { - pa_shard_t *shard = - (pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai)); - ehooks_t *ehooks = pa_shard_ehooks_get(shard); - ecache_dalloc(tsdn, &shard->pac, ehooks, &shard->pac.ecache_dirty, - edata); -} - void pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, bool *generated_dirty) { @@ -241,138 +123,10 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, } edata_szind_set(edata, SC_NSIZES); pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE); - pai_dalloc(tsdn, &shard->ecache_pai, edata); + pai_dalloc(tsdn, &shard->pac.pai, edata); *generated_dirty = true; } -static size_t -pa_stash_decayed(tsdn_t *tsdn, pa_shard_t *shard, ecache_t *ecache, - size_t npages_limit, size_t npages_decay_max, - edata_list_inactive_t *result) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); - ehooks_t *ehooks = pa_shard_ehooks_get(shard); - - /* Stash extents according to npages_limit. */ - size_t nstashed = 0; - while (nstashed < npages_decay_max) { - edata_t *edata = ecache_evict(tsdn, &shard->pac, ehooks, ecache, - npages_limit); - if (edata == NULL) { - break; - } - edata_list_inactive_append(result, edata); - nstashed += edata_size_get(edata) >> LG_PAGE; - } - return nstashed; -} - -static size_t -pa_decay_stashed(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay, - pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay, - edata_list_inactive_t *decay_extents) { - bool err; - - size_t nmadvise = 0; - size_t nunmapped = 0; - size_t npurged = 0; - - ehooks_t *ehooks = pa_shard_ehooks_get(shard); - - bool try_muzzy = !fully_decay && pa_shard_may_have_muzzy(shard); - - for (edata_t *edata = edata_list_inactive_first(decay_extents); - edata != NULL; edata = edata_list_inactive_first(decay_extents)) { - edata_list_inactive_remove(decay_extents, edata); - - size_t size = edata_size_get(edata); - size_t npages = size >> LG_PAGE; - - nmadvise++; - npurged += npages; - - switch (ecache->state) { - case extent_state_active: - not_reached(); - case extent_state_dirty: - if (try_muzzy) { - err = extent_purge_lazy_wrapper(tsdn, ehooks, - edata, /* offset */ 0, size); - if (!err) { - ecache_dalloc(tsdn, &shard->pac, ehooks, - &shard->pac.ecache_muzzy, edata); - break; - } - } - JEMALLOC_FALLTHROUGH; - case extent_state_muzzy: - extent_dalloc_wrapper(tsdn, &shard->pac, ehooks, edata); - nunmapped += npages; - break; - case extent_state_retained: - default: - not_reached(); - } - } - - if (config_stats) { - LOCKEDINT_MTX_LOCK(tsdn, *shard->stats_mtx); - locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &decay_stats->npurge, 1); - locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &decay_stats->nmadvise, nmadvise); - locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx), - &decay_stats->purged, npurged); - LOCKEDINT_MTX_UNLOCK(tsdn, *shard->stats_mtx); - atomic_fetch_sub_zu(&shard->pac.stats->pac_mapped, - nunmapped << LG_PAGE, ATOMIC_RELAXED); - } - - return npurged; -} - -/* - * npages_limit: Decay at most npages_decay_max pages without violating the - * invariant: (ecache_npages_get(ecache) >= npages_limit). We need an upper - * bound on number of pages in order to prevent unbounded growth (namely in - * stashed), otherwise unbounded new pages could be added to extents during the - * current decay run, so that the purging thread never finishes. - */ -static void -pa_decay_to_limit(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay, - pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay, - size_t npages_limit, size_t npages_decay_max) { - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 1); - - if (decay->purging || npages_decay_max == 0) { - return; - } - decay->purging = true; - malloc_mutex_unlock(tsdn, &decay->mtx); - - edata_list_inactive_t decay_extents; - edata_list_inactive_init(&decay_extents); - size_t npurge = pa_stash_decayed(tsdn, shard, ecache, npages_limit, - npages_decay_max, &decay_extents); - if (npurge != 0) { - size_t npurged = pa_decay_stashed(tsdn, shard, decay, - decay_stats, ecache, fully_decay, &decay_extents); - assert(npurged == npurge); - } - - malloc_mutex_lock(tsdn, &decay->mtx); - decay->purging = false; -} - -void -pa_decay_all(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay, - pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay) { - malloc_mutex_assert_owner(tsdn, &decay->mtx); - pa_decay_to_limit(tsdn, shard, decay, decay_stats, ecache, fully_decay, - /* npages_limit */ 0, ecache_npages_get(ecache)); -} - bool pa_shard_retain_grow_limit_get_set(tsdn_t *tsdn, pa_shard_t *shard, size_t *old_limit, size_t *new_limit) { diff --git a/src/pac.c b/src/pac.c index ed17a2f5..a4370888 100644 --- a/src/pac.c +++ b/src/pac.c @@ -3,6 +3,14 @@ #include "jemalloc/internal/pac.h" +static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, + size_t alignment, bool zero); +static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, + size_t old_size, size_t new_size, bool zero); +static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, + size_t old_size, size_t new_size); +static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata); + static ehooks_t * pac_ehooks_get(pac_t *pac) { return base_ehooks_get(pac->base); @@ -27,8 +35,6 @@ pac_decay_data_get(pac_t *pac, extent_state_t state, } } - - bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms, @@ -78,9 +84,113 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, pac->stats = pac_stats; pac->stats_mtx = stats_mtx; atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED); + + pac->pai.alloc = &pac_alloc_impl; + pac->pai.expand = &pac_expand_impl; + pac->pai.shrink = &pac_shrink_impl; + pac->pai.dalloc = &pac_dalloc_impl; + return false; } +static inline bool +pac_may_have_muzzy(pac_t *pac) { + return pac_decay_ms_get(pac, extent_state_muzzy) != 0; +} + +static edata_t * +pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, + bool zero) { + pac_t *pac = (pac_t *)self; + + ehooks_t *ehooks = pac_ehooks_get(pac); + edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, + NULL, size, alignment, zero); + + if (edata == NULL && pac_may_have_muzzy(pac)) { + edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, + NULL, size, alignment, zero); + } + if (edata == NULL) { + edata = ecache_alloc_grow(tsdn, pac, ehooks, + &pac->ecache_retained, NULL, size, alignment, zero); + if (config_stats && edata != NULL) { + atomic_fetch_add_zu(&pac->stats->pac_mapped, size, + ATOMIC_RELAXED); + } + } + return edata; +} + +static bool +pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, + size_t new_size, bool zero) { + pac_t *pac = (pac_t *)self; + + ehooks_t *ehooks = pac_ehooks_get(pac); + void *trail_begin = edata_past_get(edata); + + size_t mapped_add = 0; + size_t expand_amount = new_size - old_size; + + if (ehooks_merge_will_fail(ehooks)) { + return true; + } + edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, + trail_begin, expand_amount, PAGE, zero); + if (trail == NULL) { + trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, + trail_begin, expand_amount, PAGE, zero); + } + if (trail == NULL) { + trail = ecache_alloc_grow(tsdn, pac, ehooks, + &pac->ecache_retained, trail_begin, expand_amount, PAGE, + zero); + mapped_add = expand_amount; + } + if (trail == NULL) { + return true; + } + if (extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) { + extent_dalloc_wrapper(tsdn, pac, ehooks, trail); + return true; + } + if (config_stats && mapped_add > 0) { + atomic_fetch_add_zu(&pac->stats->pac_mapped, mapped_add, + ATOMIC_RELAXED); + } + return false; +} + +static bool +pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, + size_t new_size) { + pac_t *pac = (pac_t *)self; + + ehooks_t *ehooks = pac_ehooks_get(pac); + size_t shrink_amount = old_size - new_size; + + + if (ehooks_split_will_fail(ehooks)) { + return true; + } + + edata_t *trail = extent_split_wrapper(tsdn, pac, ehooks, edata, + new_size, shrink_amount); + if (trail == NULL) { + return true; + } + ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail); + return false; +} + +static void +pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata) { + pac_t *pac = (pac_t *)self; + ehooks_t *ehooks = pac_ehooks_get(pac); + ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata); +} + bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit) { @@ -107,7 +217,8 @@ pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, static size_t pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, - size_t npages_limit, size_t npages_decay_max, edata_list_t *result) { + size_t npages_limit, size_t npages_decay_max, + edata_list_inactive_t *result) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); ehooks_t *ehooks = pac_ehooks_get(pac); @@ -120,7 +231,7 @@ pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, if (edata == NULL) { break; } - edata_list_append(result, edata); + edata_list_inactive_append(result, edata); nstashed += edata_size_get(edata) >> LG_PAGE; } return nstashed; @@ -129,7 +240,7 @@ pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, static size_t pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay, pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay, - edata_list_t *decay_extents) { + edata_list_inactive_t *decay_extents) { bool err; size_t nmadvise = 0; @@ -141,9 +252,9 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay, bool try_muzzy = !fully_decay && pac_decay_ms_get(pac, extent_state_muzzy) != 0; - for (edata_t *edata = edata_list_first(decay_extents); edata != - NULL; edata = edata_list_first(decay_extents)) { - edata_list_remove(decay_extents, edata); + for (edata_t *edata = edata_list_inactive_first(decay_extents); edata != + NULL; edata = edata_list_inactive_first(decay_extents)) { + edata_list_inactive_remove(decay_extents, edata); size_t size = edata_size_get(edata); size_t npages = size >> LG_PAGE; @@ -211,8 +322,8 @@ pac_decay_to_limit(tsdn_t *tsdn, pac_t *pac, decay_t *decay, decay->purging = true; malloc_mutex_unlock(tsdn, &decay->mtx); - edata_list_t decay_extents; - edata_list_init(&decay_extents); + edata_list_inactive_t decay_extents; + edata_list_inactive_init(&decay_extents); size_t npurge = pac_stash_decayed(tsdn, pac, ecache, npages_limit, npages_decay_max, &decay_extents); if (npurge != 0) {