diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index c13d8289..b6b33ce3 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -28,7 +28,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, bin_stats_data_t *bstats, arena_stats_large_t *lstats, arena_stats_extents_t *estats); void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent); + ehooks_t *ehooks, extent_t *extent); #ifdef JEMALLOC_JET size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr); #endif @@ -72,7 +72,7 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache, hook_ralloc_args_t *hook_args); dss_prec_t arena_dss_prec_get(arena_t *arena); -extent_hooks_t *arena_get_extent_hooks(arena_t *arena); +ehooks_t *arena_get_ehooks(arena_t *arena); extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 9ccfaa90..16da67e2 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -14,11 +14,6 @@ arena_get_from_extent(extent_t *extent) { ATOMIC_RELAXED); } -JEMALLOC_ALWAYS_INLINE bool -arena_has_default_hooks(arena_t *arena) { - return (arena_get_extent_hooks(arena) == &extent_hooks_default); -} - JEMALLOC_ALWAYS_INLINE arena_t * arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { if (arena != NULL) { @@ -134,10 +129,10 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { /* Purge a single extent to retained / unmapped directly. */ JEMALLOC_ALWAYS_INLINE void -arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t *extent_hooks, +arena_decay_extent(tsdn_t *tsdn,arena_t *arena, ehooks_t *ehooks, extent_t *extent) { size_t extent_size = extent_size_get(extent); - extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent); + extent_dalloc_wrapper(tsdn, arena, ehooks, extent); if (config_stats) { /* Update stats accordingly. */ arena_stats_lock(tsdn, &arena->stats); diff --git a/include/jemalloc/internal/base_externs.h b/include/jemalloc/internal/base_externs.h index 7b705c9b..35734c3c 100644 --- a/include/jemalloc/internal/base_externs.h +++ b/include/jemalloc/internal/base_externs.h @@ -7,7 +7,7 @@ extern const char *metadata_thp_mode_names[]; base_t *b0get(void); base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); void base_delete(tsdn_t *tsdn, base_t *base); -extent_hooks_t *base_extent_hooks_get(base_t *base); +ehooks_t *base_ehooks_get(base_t *base); extent_hooks_t *base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks); void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment); diff --git a/include/jemalloc/internal/base_structs.h b/include/jemalloc/internal/base_structs.h index a3429d65..68e7896e 100644 --- a/include/jemalloc/internal/base_structs.h +++ b/include/jemalloc/internal/base_structs.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_BASE_STRUCTS_H #define JEMALLOC_INTERNAL_BASE_STRUCTS_H +#include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" @@ -23,10 +24,9 @@ struct base_s { unsigned ind; /* - * User-configurable extent hook functions. Points to an - * extent_hooks_t. + * User-configurable extent hook functions. */ - atomic_p_t extent_hooks; + ehooks_t ehooks; /* Protects base_alloc() and base_stats_get() operations. */ malloc_mutex_t mtx; diff --git a/include/jemalloc/internal/ehooks.h b/include/jemalloc/internal/ehooks.h index 695859d5..c79ea24b 100644 --- a/include/jemalloc/internal/ehooks.h +++ b/include/jemalloc/internal/ehooks.h @@ -1,4 +1,151 @@ #ifndef JEMALLOC_INTERNAL_EHOOKS_H #define JEMALLOC_INTERNAL_EHOOKS_H +#include "jemalloc/internal/atomic.h" + +extern const extent_hooks_t extent_hooks_default; + +typedef struct ehooks_s ehooks_t; +struct ehooks_s { + /* Logically an extent_hooks_t *. */ + atomic_p_t ptr; +}; + +void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks); + +static inline void +ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) { + atomic_store_p(&ehooks->ptr, extent_hooks, ATOMIC_RELEASE); +} + +static inline extent_hooks_t * +ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) { + return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE); +} + +static inline bool +ehooks_are_default(ehooks_t *ehooks) { + return ehooks_get_extent_hooks_ptr(ehooks) == &extent_hooks_default; +} + +static inline bool +ehooks_destroy_is_noop(ehooks_t *ehooks) { + return ehooks_get_extent_hooks_ptr(ehooks)->destroy == NULL; +} + +static inline bool +ehooks_purge_lazy_will_fail(ehooks_t *ehooks) { + return ehooks_get_extent_hooks_ptr(ehooks)->purge_lazy == NULL; +} + +static inline bool +ehooks_purge_forced_will_fail(ehooks_t *ehooks) { + return ehooks_get_extent_hooks_ptr(ehooks)->purge_forced == NULL; +} + +static inline bool +ehooks_split_will_fail(ehooks_t *ehooks) { + return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL; +} + +static inline bool +ehooks_merge_will_fail(ehooks_t *ehooks) { + return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL; +} + +static inline void * +ehooks_alloc(ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment, + bool *zero, bool *commit, unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + return extent_hooks->alloc(extent_hooks, new_addr, size, alignment, + zero, commit, arena_ind); +} + +static inline bool +ehooks_dalloc(ehooks_t *ehooks, void *addr, size_t size, bool committed, + unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks->dalloc == NULL) { + return true; + } + return extent_hooks->dalloc(extent_hooks, addr, size, committed, + arena_ind); +} + +static inline void +ehooks_destroy(ehooks_t *ehooks, void *addr, size_t size, bool committed, + unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks->destroy == NULL) { + return; + } + extent_hooks->destroy(extent_hooks, addr, size, committed, arena_ind); +} + +static inline bool +ehooks_commit(ehooks_t *ehooks, void *addr, size_t size, size_t offset, + size_t length, unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks->commit == NULL) { + return true; + } + return extent_hooks->commit(extent_hooks, addr, size, offset, length, + arena_ind); +} + +static inline bool +ehooks_decommit(ehooks_t *ehooks, void *addr, size_t size, size_t offset, + size_t length, unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks->decommit == NULL) { + return true; + } + return extent_hooks->decommit(extent_hooks, addr, size, offset, length, + arena_ind); +} + +static inline bool +ehooks_purge_lazy(ehooks_t *ehooks, void *addr, size_t size, size_t offset, + size_t length, unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks->purge_lazy == NULL) { + return true; + } + return extent_hooks->purge_lazy(extent_hooks, addr, size, offset, + length, arena_ind); +} + +static inline bool +ehooks_purge_forced(ehooks_t *ehooks, void *addr, size_t size, size_t offset, + size_t length, unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks->purge_forced == NULL) { + return true; + } + return extent_hooks->purge_forced(extent_hooks, addr, size, offset, + length, arena_ind); +} + +static inline bool +ehooks_split(ehooks_t *ehooks, void *addr, size_t size, size_t size_a, + size_t size_b, bool committed, unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks->split == NULL) { + return true; + } + return extent_hooks->split(extent_hooks, addr, size, size_a, size_b, + committed, arena_ind); +} + +static inline bool +ehooks_merge(ehooks_t *ehooks, void *addr_a, size_t size_a, void *addr_b, + size_t size_b, bool committed, unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks->merge == NULL) { + return true; + } + return extent_hooks->merge(extent_hooks, addr_a, size_a, addr_b, size_b, + committed, arena_ind); +} + #endif /* JEMALLOC_INTERNAL_EHOOKS_H */ diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent_externs.h index 218ca94f..26828ba5 100644 --- a/include/jemalloc/internal/extent_externs.h +++ b/include/jemalloc/internal/extent_externs.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_EXTENT_EXTERNS_H #define JEMALLOC_INTERNAL_EXTENT_EXTERNS_H +#include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex_pool.h" #include "jemalloc/internal/ph.h" @@ -9,7 +10,6 @@ extern size_t opt_lg_extent_max_active_fit; extern rtree_t extents_rtree; -extern const extent_hooks_t extent_hooks_default; extern mutex_pool_t extent_mutex_pool; extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); @@ -18,39 +18,34 @@ void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent); ph_proto(, extent_avail_, extent_tree_t, extent_t) ph_proto(, extent_heap_, extent_heap_t, extent_t) -extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, eset_t *eset, void *new_addr, - size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, - bool *zero, bool *commit); -void extents_dalloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, eset_t *eset, extent_t *extent); -extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, eset_t *eset, size_t npages_min); -extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad, - size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit); +extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment, + bool slab, szind_t szind, bool *zero, bool *commit); +void extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + eset_t *eset, extent_t *extent); +extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + eset_t *eset, size_t npages_min); +extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, + szind_t szind, bool *zero, bool *commit); void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent); -void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent); -void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent); -bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length); -bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length); -bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length); -bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length); -extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, - szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b); -bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *a, extent_t *b); +void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent); +void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent); +bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length); +bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length); +bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length); +bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length); +extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a, + size_t size_b, szind_t szind_b, bool slab_b); +bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *a, extent_t *b); bool extent_boot(void); diff --git a/src/arena.c b/src/arena.c index a272438a..214a97c2 100644 --- a/src/arena.c +++ b/src/arena.c @@ -4,6 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/div.h" +#include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" @@ -252,13 +253,12 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, } void -arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent) { +arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - extents_dalloc(tsdn, arena, extent_hooks, &arena->eset_dirty, - extent); + extents_dalloc(tsdn, arena, ehooks, &arena->eset_dirty, extent); if (arena_dirty_decay_ms_get(arena) == 0) { arena_decay_dirty(tsdn, arena, false, true); } else { @@ -426,7 +426,7 @@ arena_may_have_muzzy(arena_t *arena) { extent_t * arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero) { - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); + ehooks_t *ehooks = arena_get_ehooks(arena); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -434,19 +434,18 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, szind_t szind = sz_size2index(usize); size_t mapped_add; bool commit = true; - extent_t *extent = extents_alloc(tsdn, arena, extent_hooks, + extent_t *extent = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); if (extent == NULL && arena_may_have_muzzy(arena)) { - extent = extents_alloc(tsdn, arena, extent_hooks, - &arena->eset_muzzy, NULL, usize, sz_large_pad, alignment, - false, szind, zero, &commit); + extent = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy, + NULL, usize, sz_large_pad, alignment, false, szind, zero, + &commit); } size_t size = usize + sz_large_pad; if (extent == NULL) { - extent = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, - usize, sz_large_pad, alignment, false, szind, zero, - &commit); + extent = extent_alloc_wrapper(tsdn, arena, ehooks, NULL, usize, + sz_large_pad, alignment, false, szind, zero, &commit); if (config_stats) { /* * extent may be NULL on OOM, but in that case @@ -819,8 +818,8 @@ arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena, static size_t arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, eset_t *eset, size_t npages_limit, - size_t npages_decay_max, extent_list_t *decay_extents) { + ehooks_t *ehooks, eset_t *eset, size_t npages_limit, + size_t npages_decay_max, extent_list_t *decay_extents) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -828,8 +827,8 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, size_t nstashed = 0; extent_t *extent; while (nstashed < npages_decay_max && - (extent = extents_evict(tsdn, arena, extent_hooks, eset, - npages_limit)) != NULL) { + (extent = extents_evict(tsdn, arena, ehooks, eset, npages_limit)) + != NULL) { extent_list_append(decay_extents, extent); nstashed += extent_size_get(extent) >> LG_PAGE; } @@ -837,9 +836,9 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, } static size_t -arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, arena_decay_t *decay, eset_t *eset, - bool all, extent_list_t *decay_extents, bool is_background_thread) { +arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + arena_decay_t *decay, eset_t *eset, bool all, extent_list_t *decay_extents, + bool is_background_thread) { size_t nmadvise, nunmapped; size_t npurged; @@ -864,9 +863,9 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, case extent_state_dirty: if (!all && muzzy_decay_ms != 0 && !extent_purge_lazy_wrapper(tsdn, arena, - extent_hooks, extent, 0, + ehooks, extent, 0, extent_size_get(extent))) { - extents_dalloc(tsdn, arena, extent_hooks, + extents_dalloc(tsdn, arena, ehooks, &arena->eset_muzzy, extent); arena_background_thread_inactivity_check(tsdn, arena, is_background_thread); @@ -874,8 +873,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, } JEMALLOC_FALLTHROUGH; case extent_state_muzzy: - extent_dalloc_wrapper(tsdn, arena, extent_hooks, - extent); + extent_dalloc_wrapper(tsdn, arena, ehooks, extent); if (config_stats) { nunmapped += npages; } @@ -923,17 +921,16 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, decay->purging = true; malloc_mutex_unlock(tsdn, &decay->mtx); - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); + ehooks_t *ehooks = arena_get_ehooks(arena); extent_list_t decay_extents; extent_list_init(&decay_extents); - size_t npurge = arena_stash_decayed(tsdn, arena, extent_hooks, eset, + size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, eset, npages_limit, npages_decay_max, &decay_extents); if (npurge != 0) { - size_t npurged = arena_decay_stashed(tsdn, arena, - extent_hooks, decay, eset, all, &decay_extents, - is_background_thread); + size_t npurged = arena_decay_stashed(tsdn, arena, ehooks, decay, + eset, all, &decay_extents, is_background_thread); assert(npurged == npurge); } @@ -1006,8 +1003,8 @@ static void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) { arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); - arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, slab); + ehooks_t *ehooks = arena_get_ehooks(arena); + arena_extents_dirty_dalloc(tsdn, arena, ehooks, slab); } static void @@ -1159,11 +1156,11 @@ arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) { * destroyed, or provide custom extent hooks that track retained * dss-based extents for later reuse. */ - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); + ehooks_t *ehooks = arena_get_ehooks(arena); extent_t *extent; - while ((extent = extents_evict(tsdn, arena, extent_hooks, + while ((extent = extents_evict(tsdn, arena, ehooks, &arena->eset_retained, 0)) != NULL) { - extent_destroy_wrapper(tsdn, arena, extent_hooks, extent); + extent_destroy_wrapper(tsdn, arena, ehooks, extent); } } @@ -1204,9 +1201,8 @@ arena_destroy(tsd_t *tsd, arena_t *arena) { } static extent_t * -arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, const bin_info_t *bin_info, - szind_t szind) { +arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + const bin_info_t *bin_info, szind_t szind) { extent_t *slab; bool zero, commit; @@ -1215,7 +1211,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, zero = false; commit = true; - slab = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, + slab = extent_alloc_wrapper(tsdn, arena, ehooks, NULL, bin_info->slab_size, 0, PAGE, true, szind, &zero, &commit); if (config_stats && slab != NULL) { @@ -1232,21 +1228,20 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); + ehooks_t *ehooks = arena_get_ehooks(arena); szind_t szind = sz_size2index(bin_info->reg_size); bool zero = false; bool commit = true; - extent_t *slab = extents_alloc(tsdn, arena, extent_hooks, - &arena->eset_dirty, NULL, bin_info->slab_size, 0, PAGE, true, - binind, &zero, &commit); + extent_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty, + NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); if (slab == NULL && arena_may_have_muzzy(arena)) { - slab = extents_alloc(tsdn, arena, extent_hooks, - &arena->eset_muzzy, NULL, bin_info->slab_size, 0, PAGE, - true, binind, &zero, &commit); + slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy, + NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, + &commit); } if (slab == NULL) { - slab = arena_slab_alloc_hard(tsdn, arena, extent_hooks, - bin_info, szind); + slab = arena_slab_alloc_hard(tsdn, arena, ehooks, bin_info, + szind); if (slab == NULL) { return NULL; } @@ -1846,9 +1841,9 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, return ret; } -extent_hooks_t * -arena_get_extent_hooks(arena_t *arena) { - return base_extent_hooks_get(arena->base); +ehooks_t * +arena_get_ehooks(arena_t *arena) { + return base_ehooks_get(arena->base); } extent_hooks_t * diff --git a/src/base.c b/src/base.c index 9a55ed2e..92dfca8c 100644 --- a/src/base.c +++ b/src/base.c @@ -29,7 +29,7 @@ metadata_thp_madvise(void) { } static void * -base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) { +base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) { void *addr; bool zero = true; bool commit = true; @@ -37,7 +37,7 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) /* Use huge page sizes and alignment regardless of opt_metadata_thp. */ assert(size == HUGEPAGE_CEILING(size)); size_t alignment = HUGEPAGE; - if (extent_hooks == &extent_hooks_default) { + if (ehooks_are_default(ehooks)) { addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit); if (have_madvise_huge && addr) { pages_set_thp_state(addr, size); @@ -46,8 +46,8 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) /* No arena context as we are creating new arenas. */ tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); pre_reentrancy(tsd, NULL); - addr = extent_hooks->alloc(extent_hooks, NULL, size, alignment, - &zero, &commit, ind); + addr = ehooks_alloc(ehooks, NULL, size, alignment, &zero, + &commit, ind); post_reentrancy(tsd); } @@ -55,7 +55,7 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) } static void -base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr, +base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr, size_t size) { /* * Cascade through dalloc, decommit, purge_forced, and purge_lazy, @@ -67,7 +67,7 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr, * may in fact want the end state of all associated virtual memory to be * in some consistent-but-allocated state. */ - if (extent_hooks == &extent_hooks_default) { + if (ehooks_are_default(ehooks)) { if (!extent_dalloc_mmap(addr, size)) { goto label_done; } @@ -85,24 +85,16 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr, } else { tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); pre_reentrancy(tsd, NULL); - if (extent_hooks->dalloc != NULL && - !extent_hooks->dalloc(extent_hooks, addr, size, true, - ind)) { + if (!ehooks_dalloc(ehooks, addr, size, true, ind)) { goto label_post_reentrancy; } - if (extent_hooks->decommit != NULL && - !extent_hooks->decommit(extent_hooks, addr, size, 0, size, - ind)) { + if (!ehooks_decommit(ehooks, addr, size, 0, size, ind)) { goto label_post_reentrancy; } - if (extent_hooks->purge_forced != NULL && - !extent_hooks->purge_forced(extent_hooks, addr, size, 0, - size, ind)) { + if (!ehooks_purge_forced(ehooks, addr, size, 0, size, ind)) { goto label_post_reentrancy; } - if (extent_hooks->purge_lazy != NULL && - !extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size, - ind)) { + if (!ehooks_purge_lazy(ehooks, addr, size, 0, size, ind)) { goto label_post_reentrancy; } /* Nothing worked. That's the application's problem. */ @@ -248,8 +240,8 @@ base_extent_bump_alloc(base_t *base, extent_t *extent, size_t size, * On success a pointer to the initialized base_block_t header is returned. */ static base_block_t * -base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks, - unsigned ind, pszind_t *pind_last, size_t *extent_sn_next, size_t size, +base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind, + pszind_t *pind_last, size_t *extent_sn_next, size_t size, size_t alignment) { alignment = ALIGNMENT_CEILING(alignment, QUANTUM); size_t usize = ALIGNMENT_CEILING(size, alignment); @@ -270,7 +262,7 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks, size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next)); size_t block_size = (min_block_size > next_block_size) ? min_block_size : next_block_size; - base_block_t *block = (base_block_t *)base_map(tsdn, extent_hooks, ind, + base_block_t *block = (base_block_t *)base_map(tsdn, ehooks, ind, block_size); if (block == NULL) { return NULL; @@ -311,13 +303,13 @@ static extent_t * base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) { malloc_mutex_assert_owner(tsdn, &base->mtx); - extent_hooks_t *extent_hooks = base_extent_hooks_get(base); + ehooks_t *ehooks = base_ehooks_get(base); /* * Drop mutex during base_block_alloc(), because an extent hook will be * called. */ malloc_mutex_unlock(tsdn, &base->mtx); - base_block_t *block = base_block_alloc(tsdn, base, extent_hooks, + base_block_t *block = base_block_alloc(tsdn, base, ehooks, base_ind_get(base), &base->pind_last, &base->extent_sn_next, size, alignment); malloc_mutex_lock(tsdn, &base->mtx); @@ -353,7 +345,16 @@ base_t * base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { pszind_t pind_last = 0; size_t extent_sn_next = 0; - base_block_t *block = base_block_alloc(tsdn, NULL, extent_hooks, ind, + + /* + * The base will contain the ehooks eventually, but it itself is + * allocated using them. So we use some stack ehooks to bootstrap its + * memory, and then initialize the ehooks within the base_t. + */ + ehooks_t fake_ehooks; + ehooks_init(&fake_ehooks, extent_hooks); + + base_block_t *block = base_block_alloc(tsdn, NULL, &fake_ehooks, ind, &pind_last, &extent_sn_next, sizeof(base_t), QUANTUM); if (block == NULL) { return NULL; @@ -365,10 +366,10 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { base_t *base = (base_t *)base_extent_bump_alloc_helper(&block->extent, &gap_size, base_size, base_alignment); base->ind = ind; - atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELAXED); + ehooks_init(&base->ehooks, extent_hooks); if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE, malloc_mutex_rank_exclusive)) { - base_unmap(tsdn, extent_hooks, ind, block, block->size); + base_unmap(tsdn, &fake_ehooks, ind, block, block->size); return NULL; } base->pind_last = pind_last; @@ -397,26 +398,26 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { void base_delete(tsdn_t *tsdn, base_t *base) { - extent_hooks_t *extent_hooks = base_extent_hooks_get(base); + ehooks_t *ehooks = base_ehooks_get(base); base_block_t *next = base->blocks; do { base_block_t *block = next; next = block->next; - base_unmap(tsdn, extent_hooks, base_ind_get(base), block, + base_unmap(tsdn, ehooks, base_ind_get(base), block, block->size); } while (next != NULL); } -extent_hooks_t * -base_extent_hooks_get(base_t *base) { - return (extent_hooks_t *)atomic_load_p(&base->extent_hooks, - ATOMIC_ACQUIRE); +ehooks_t * +base_ehooks_get(base_t *base) { + return &base->ehooks; } extent_hooks_t * base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks) { - extent_hooks_t *old_extent_hooks = base_extent_hooks_get(base); - atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELEASE); + extent_hooks_t *old_extent_hooks = + ehooks_get_extent_hooks_ptr(&base->ehooks); + ehooks_init(&base->ehooks, extent_hooks); return old_extent_hooks; } diff --git a/src/ctl.c b/src/ctl.c index c2f12704..9b88f403 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -2400,8 +2400,9 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, arena, new_extent_hooks); READ(old_extent_hooks, extent_hooks_t *); } else { - old_extent_hooks = arena_get_extent_hooks( - arena); + old_extent_hooks = + ehooks_get_extent_hooks_ptr( + arena_get_ehooks(arena)); READ(old_extent_hooks, extent_hooks_t *); } } diff --git a/src/ehooks.c b/src/ehooks.c index 454cb475..0f59f339 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -1,3 +1,8 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" +#include "jemalloc/internal/ehooks.h" + +void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks) { + ehooks_set_extent_hooks_ptr(ehooks, extent_hooks); +} diff --git a/src/extent.c b/src/extent.c index d21a1e81..23194e1d 100644 --- a/src/extent.c +++ b/src/extent.c @@ -28,40 +28,38 @@ static void extent_destroy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind); static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); -static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length, bool growing_retained); -static bool extent_decommit_default(extent_hooks_t *extent_hooks, - void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); +static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length, bool growing_retained); +static bool extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, + size_t size, size_t offset, size_t length, unsigned arena_ind); #ifdef PAGES_CAN_PURGE_LAZY static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); #endif static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length, bool growing_retained); + ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, + bool growing_retained); #ifdef PAGES_CAN_PURGE_FORCED static bool extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); #endif static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length, bool growing_retained); + ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, + bool growing_retained); static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind); static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, - szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b, + ehooks_t *ehooks, extent_t *extent, size_t size_a, szind_t szind_a, + bool slab_a, size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained); static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind); -static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *a, extent_t *b, - bool growing_retained); +static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *a, extent_t *b, bool growing_retained); -const extent_hooks_t extent_hooks_default = { +const extent_hooks_t extent_hooks_default = { extent_alloc_default, extent_dalloc_default, extent_destroy_default, @@ -97,16 +95,14 @@ static atomic_zu_t highpages; */ static void extent_deregister(tsdn_t *tsdn, extent_t *extent); -static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, eset_t *eset, void *new_addr, - size_t usize, size_t pad, size_t alignment, bool slab, szind_t szind, - bool *zero, bool *commit, bool growing_retained); +static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + eset_t *eset, void *new_addr, size_t usize, size_t pad, size_t alignment, + bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained); static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, - extent_t *extent, bool *coalesced, bool growing_retained); -static void extent_record(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, eset_t *eset, extent_t *extent, - bool growing_retained); + ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, + bool *coalesced, bool growing_retained); +static void extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + eset_t *eset, extent_t *extent, bool growing_retained); /******************************************************************************/ @@ -224,13 +220,12 @@ extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp) static bool -extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, - extent_t *extent) { +extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent) { extent_state_set(extent, extent_state_active); bool coalesced; - extent = extent_try_coalesce(tsdn, arena, extent_hooks, rtree_ctx, - eset, extent, &coalesced, false); + extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, eset, + extent, &coalesced, false); extent_state_set(extent, eset_state_get(eset)); if (!coalesced) { @@ -241,23 +236,23 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, } extent_t * -extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - eset_t *eset, void *new_addr, size_t size, size_t pad, - size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { +extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, + void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, + szind_t szind, bool *zero, bool *commit) { assert(size + pad != 0); assert(alignment != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - extent_t *extent = extent_recycle(tsdn, arena, extent_hooks, eset, - new_addr, size, pad, alignment, slab, szind, zero, commit, false); + extent_t *extent = extent_recycle(tsdn, arena, ehooks, eset, new_addr, + size, pad, alignment, slab, szind, zero, commit, false); assert(extent == NULL || extent_dumpable_get(extent)); return extent; } void -extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - eset_t *eset, extent_t *extent) { +extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, + extent_t *extent) { assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); assert(extent_dumpable_get(extent)); @@ -267,12 +262,12 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_addr_set(extent, extent_base_get(extent)); extent_zeroed_set(extent, false); - extent_record(tsdn, arena, extent_hooks, eset, extent, false); + extent_record(tsdn, arena, ehooks, eset, extent, false); } extent_t * -extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - eset_t *eset, size_t npages_min) { +extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, + size_t npages_min) { rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -301,8 +296,8 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, break; } /* Try to coalesce. */ - if (extent_try_delayed_coalesce(tsdn, arena, extent_hooks, - rtree_ctx, eset, extent)) { + if (extent_try_delayed_coalesce(tsdn, arena, ehooks, rtree_ctx, + eset, extent)) { break; } /* @@ -339,8 +334,8 @@ label_return: * indicates OOM), e.g. when trying to split an existing extent. */ static void -extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - eset_t *eset, extent_t *extent, bool growing_retained) { +extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, + extent_t *extent, bool growing_retained) { size_t sz = extent_size_get(extent); if (config_stats) { arena_stats_accum_zu(&arena->stats.abandoned_vm, sz); @@ -350,11 +345,10 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, * that this is only a virtual memory leak. */ if (eset_state_get(eset) == extent_state_dirty) { - if (extent_purge_lazy_impl(tsdn, arena, extent_hooks, - extent, 0, sz, growing_retained)) { - extent_purge_forced_impl(tsdn, arena, extent_hooks, - extent, 0, extent_size_get(extent), - growing_retained); + if (extent_purge_lazy_impl(tsdn, arena, ehooks, extent, 0, sz, + growing_retained)) { + extent_purge_forced_impl(tsdn, arena, ehooks, extent, 0, + extent_size_get(extent), growing_retained); } } extent_dalloc(tsdn, arena, extent); @@ -581,10 +575,9 @@ extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) { * given allocation request. */ static extent_t * -extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, - void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, - bool growing_retained) { +extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size, + size_t pad, size_t alignment, bool slab, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); assert(alignment > 0); @@ -668,8 +661,8 @@ typedef enum { } extent_split_interior_result_t; static extent_split_interior_result_t -extent_split_interior(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, +extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + rtree_ctx_t *rtree_ctx, /* The result of splitting, in case of success. */ extent_t **extent, extent_t **lead, extent_t **trail, /* The mess to clean up, in case of error. */ @@ -693,9 +686,9 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, /* Split the lead. */ if (leadsize != 0) { *lead = *extent; - *extent = extent_split_impl(tsdn, arena, extent_hooks, - *lead, leadsize, SC_NSIZES, false, esize + trailsize, szind, - slab, growing_retained); + *extent = extent_split_impl(tsdn, arena, ehooks, *lead, + leadsize, SC_NSIZES, false, esize + trailsize, szind, slab, + growing_retained); if (*extent == NULL) { *to_leak = *lead; *lead = NULL; @@ -705,9 +698,8 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, /* Split the trail. */ if (trailsize != 0) { - *trail = extent_split_impl(tsdn, arena, extent_hooks, *extent, - esize, szind, slab, trailsize, SC_NSIZES, false, - growing_retained); + *trail = extent_split_impl(tsdn, arena, ehooks, *extent, esize, + szind, slab, trailsize, SC_NSIZES, false, growing_retained); if (*trail == NULL) { *to_leak = *extent; *to_salvage = *lead; @@ -745,18 +737,18 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, * and put back into eset. */ static extent_t * -extent_recycle_split(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, - void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, - szind_t szind, extent_t *extent, bool growing_retained) { +extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size, + size_t pad, size_t alignment, bool slab, szind_t szind, extent_t *extent, + bool growing_retained) { extent_t *lead; extent_t *trail; extent_t *to_leak; extent_t *to_salvage; extent_split_interior_result_t result = extent_split_interior( - tsdn, arena, extent_hooks, rtree_ctx, &extent, &lead, &trail, - &to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind, + tsdn, arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak, + &to_salvage, new_addr, size, pad, alignment, slab, szind, growing_retained); if (!maps_coalesce && result != extent_split_interior_ok @@ -790,8 +782,8 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, if (to_leak != NULL) { void *leak = extent_base_get(to_leak); extent_deregister_no_gdump_sub(tsdn, to_leak); - extents_abandon_vm(tsdn, arena, extent_hooks, eset, - to_leak, growing_retained); + extents_abandon_vm(tsdn, arena, ehooks, eset, to_leak, + growing_retained); assert(extent_lock_from_addr(tsdn, rtree_ctx, leak, false) == NULL); } @@ -807,7 +799,7 @@ extent_need_manual_zero(arena_t *arena) { * default extent hooks installed (in which case the purge semantics may * change); or 2) transparent huge pages enabled. */ - return (!arena_has_default_hooks(arena) || + return (!ehooks_are_default(arena_get_ehooks(arena)) || (opt_thp == thp_mode_always)); } @@ -816,10 +808,9 @@ extent_need_manual_zero(arena_t *arena) { * in the given eset_t. */ static extent_t * -extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - eset_t *eset, void *new_addr, size_t size, size_t pad, - size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit, - bool growing_retained) { +extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, + void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, + szind_t szind, bool *zero, bool *commit, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); assert(new_addr == NULL || !slab); @@ -829,25 +820,25 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - extent_t *extent = extent_recycle_extract(tsdn, arena, extent_hooks, + extent_t *extent = extent_recycle_extract(tsdn, arena, ehooks, rtree_ctx, eset, new_addr, size, pad, alignment, slab, growing_retained); if (extent == NULL) { return NULL; } - extent = extent_recycle_split(tsdn, arena, extent_hooks, rtree_ctx, - eset, new_addr, size, pad, alignment, slab, szind, extent, + extent = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, eset, + new_addr, size, pad, alignment, slab, szind, extent, growing_retained); if (extent == NULL) { return NULL; } if (*commit && !extent_committed_get(extent)) { - if (extent_commit_impl(tsdn, arena, extent_hooks, extent, - 0, extent_size_get(extent), growing_retained)) { - extent_record(tsdn, arena, extent_hooks, eset, - extent, growing_retained); + if (extent_commit_impl(tsdn, arena, ehooks, extent, 0, + extent_size_get(extent), growing_retained)) { + extent_record(tsdn, arena, ehooks, eset, extent, + growing_retained); return NULL; } if (!extent_need_manual_zero(arena)) { @@ -985,9 +976,9 @@ extent_hook_post_reentrancy(tsdn_t *tsdn) { * virtual memory ranges retained by each arena. */ static extent_t * -extent_grow_retained(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, size_t size, size_t pad, size_t alignment, - bool slab, szind_t szind, bool *zero, bool *commit) { +extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, + bool *zero, bool *commit) { malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx); assert(pad == 0 || !slab); assert(!*zero || !slab); @@ -1022,14 +1013,13 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, bool committed = false; void *ptr; - if (extent_hooks == &extent_hooks_default) { + if (ehooks_are_default(ehooks)) { ptr = extent_alloc_default_impl(tsdn, arena, NULL, alloc_size, PAGE, &zeroed, &committed); } else { extent_hook_pre_reentrancy(tsdn, arena); - ptr = extent_hooks->alloc(extent_hooks, NULL, - alloc_size, PAGE, &zeroed, &committed, - arena_ind_get(arena)); + ptr = ehooks_alloc(ehooks, NULL, alloc_size, PAGE, &zeroed, + &committed, arena_ind_get(arena)); extent_hook_post_reentrancy(tsdn); } @@ -1060,18 +1050,17 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, extent_t *trail; extent_t *to_leak; extent_t *to_salvage; - extent_split_interior_result_t result = extent_split_interior( - tsdn, arena, extent_hooks, rtree_ctx, &extent, &lead, &trail, - &to_leak, &to_salvage, NULL, size, pad, alignment, slab, szind, - true); + extent_split_interior_result_t result = extent_split_interior(tsdn, + arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak, + &to_salvage, NULL, size, pad, alignment, slab, szind, true); if (result == extent_split_interior_ok) { if (lead != NULL) { - extent_record(tsdn, arena, extent_hooks, + extent_record(tsdn, arena, ehooks, &arena->eset_retained, lead, true); } if (trail != NULL) { - extent_record(tsdn, arena, extent_hooks, + extent_record(tsdn, arena, ehooks, &arena->eset_retained, trail, true); } } else { @@ -1084,21 +1073,21 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, if (config_prof) { extent_gdump_add(tsdn, to_salvage); } - extent_record(tsdn, arena, extent_hooks, + extent_record(tsdn, arena, ehooks, &arena->eset_retained, to_salvage, true); } if (to_leak != NULL) { extent_deregister_no_gdump_sub(tsdn, to_leak); - extents_abandon_vm(tsdn, arena, extent_hooks, + extents_abandon_vm(tsdn, arena, ehooks, &arena->eset_retained, to_leak, true); } goto label_err; } if (*commit && !extent_committed_get(extent)) { - if (extent_commit_impl(tsdn, arena, extent_hooks, extent, 0, + if (extent_commit_impl(tsdn, arena, ehooks, extent, 0, extent_size_get(extent), true)) { - extent_record(tsdn, arena, extent_hooks, + extent_record(tsdn, arena, ehooks, &arena->eset_retained, extent, true); goto label_err; } @@ -1151,15 +1140,15 @@ label_err: } static extent_t * -extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad, - size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { +extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, + szind_t szind, bool *zero, bool *commit) { assert(size != 0); assert(alignment != 0); malloc_mutex_lock(tsdn, &arena->extent_grow_mtx); - extent_t *extent = extent_recycle(tsdn, arena, extent_hooks, + extent_t *extent = extent_recycle(tsdn, arena, ehooks, &arena->eset_retained, new_addr, size, pad, alignment, slab, szind, zero, commit, true); if (extent != NULL) { @@ -1168,8 +1157,8 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, extent_gdump_add(tsdn, extent); } } else if (opt_retain && new_addr == NULL) { - extent = extent_grow_retained(tsdn, arena, extent_hooks, size, - pad, alignment, slab, szind, zero, commit); + extent = extent_grow_retained(tsdn, arena, ehooks, size, pad, + alignment, slab, szind, zero, commit); /* extent_grow_retained() always releases extent_grow_mtx. */ } else { malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); @@ -1180,9 +1169,9 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, } static extent_t * -extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad, - size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { +extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, + szind_t szind, bool *zero, bool *commit) { size_t esize = size + pad; extent_t *extent = extent_alloc(tsdn, arena); if (extent == NULL) { @@ -1190,14 +1179,14 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, } void *addr; size_t palignment = ALIGNMENT_CEILING(alignment, PAGE); - if (extent_hooks == &extent_hooks_default) { + if (ehooks_are_default(ehooks)) { /* Call directly to propagate tsdn. */ addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize, palignment, zero, commit); } else { extent_hook_pre_reentrancy(tsdn, arena); - addr = extent_hooks->alloc(extent_hooks, new_addr, - esize, palignment, zero, commit, arena_ind_get(arena)); + addr = ehooks_alloc(ehooks, new_addr, esize, palignment, zero, + commit, arena_ind_get(arena)); extent_hook_post_reentrancy(tsdn); } if (addr == NULL) { @@ -1219,14 +1208,14 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, } extent_t * -extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad, - size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { +extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, + szind_t szind, bool *zero, bool *commit) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - extent_t *extent = extent_alloc_retained(tsdn, arena, extent_hooks, - new_addr, size, pad, alignment, slab, szind, zero, commit); + extent_t *extent = extent_alloc_retained(tsdn, arena, ehooks, new_addr, + size, pad, alignment, slab, szind, zero, commit); if (extent == NULL) { if (opt_retain && new_addr != NULL) { /* @@ -1237,7 +1226,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, */ return NULL; } - extent = extent_alloc_wrapper_hard(tsdn, arena, extent_hooks, + extent = extent_alloc_wrapper_hard(tsdn, arena, ehooks, new_addr, size, pad, alignment, slab, szind, zero, commit); } @@ -1266,15 +1255,14 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const extent_t *inner, } static bool -extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - eset_t *eset, extent_t *inner, extent_t *outer, bool forward, - bool growing_retained) { +extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, + extent_t *inner, extent_t *outer, bool forward, bool growing_retained) { assert(extent_can_coalesce(arena, eset, inner, outer)); extent_activate_locked(tsdn, arena, eset, outer); malloc_mutex_unlock(tsdn, &eset->mtx); - bool err = extent_merge_impl(tsdn, arena, extent_hooks, + bool err = extent_merge_impl(tsdn, arena, ehooks, forward ? inner : outer, forward ? outer : inner, growing_retained); malloc_mutex_lock(tsdn, &eset->mtx); @@ -1286,10 +1274,9 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, } static extent_t * -extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, - extent_t *extent, bool *coalesced, bool growing_retained, - bool inactive_only) { +extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced, + bool growing_retained, bool inactive_only) { /* * We avoid checking / locking inactive neighbors for large size * classes, since they are eagerly coalesced on deallocation which can @@ -1318,7 +1305,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, extent_unlock(tsdn, next); if (can_coalesce && !extent_coalesce(tsdn, arena, - extent_hooks, eset, extent, next, true, + ehooks, eset, extent, next, true, growing_retained)) { if (eset->delay_coalesce) { /* Do minimal coalescing. */ @@ -1338,7 +1325,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, extent_unlock(tsdn, prev); if (can_coalesce && !extent_coalesce(tsdn, arena, - extent_hooks, eset, extent, prev, false, + ehooks, eset, extent, prev, false, growing_retained)) { extent = prev; if (eset->delay_coalesce) { @@ -1358,19 +1345,19 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, } static extent_t * -extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, - extent_t *extent, bool *coalesced, bool growing_retained) { - return extent_try_coalesce_impl(tsdn, arena, extent_hooks, rtree_ctx, - eset, extent, coalesced, growing_retained, false); +extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced, + bool growing_retained) { + return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset, + extent, coalesced, growing_retained, false); } static extent_t * -extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, - extent_t *extent, bool *coalesced, bool growing_retained) { - return extent_try_coalesce_impl(tsdn, arena, extent_hooks, rtree_ctx, - eset, extent, coalesced, growing_retained, true); +extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced, + bool growing_retained) { + return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset, + extent, coalesced, growing_retained, true); } /* @@ -1378,8 +1365,8 @@ extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, * given eset_t (coalesces, deregisters slab interiors, the heap operations). */ static void -extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - eset_t *eset, extent_t *extent, bool growing_retained) { +extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, + extent_t *extent, bool growing_retained) { rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -1399,22 +1386,22 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, (uintptr_t)extent_base_get(extent), true) == extent); if (!eset->delay_coalesce) { - extent = extent_try_coalesce(tsdn, arena, extent_hooks, - rtree_ctx, eset, extent, NULL, growing_retained); + extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, + eset, extent, NULL, growing_retained); } else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) { assert(eset == &arena->eset_dirty); /* Always coalesce large eset eagerly. */ bool coalesced; do { assert(extent_state_get(extent) == extent_state_active); - extent = extent_try_coalesce_large(tsdn, arena, - extent_hooks, rtree_ctx, eset, extent, - &coalesced, growing_retained); + extent = extent_try_coalesce_large(tsdn, arena, ehooks, + rtree_ctx, eset, extent, &coalesced, + growing_retained); } while (coalesced); if (extent_size_get(extent) >= oversize_threshold) { /* Shortcut to purge the oversize extent eagerly. */ malloc_mutex_unlock(tsdn, &eset->mtx); - arena_decay_extent(tsdn, arena, extent_hooks, extent); + arena_decay_extent(tsdn, arena, ehooks, extent); return; } } @@ -1425,7 +1412,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); + ehooks_t *ehooks = arena_get_ehooks(arena); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -1434,7 +1421,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_dalloc(tsdn, arena, extent); return; } - extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent); + extent_dalloc_wrapper(tsdn, arena, ehooks, extent); } static bool @@ -1458,8 +1445,8 @@ extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size, } static bool -extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent) { +extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent) { bool err; assert(extent_base_get(extent) != NULL); @@ -1470,16 +1457,15 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, extent_addr_set(extent, extent_base_get(extent)); /* Try to deallocate. */ - if (extent_hooks == &extent_hooks_default) { + if (ehooks_are_default(ehooks)) { /* Call directly to propagate tsdn. */ err = extent_dalloc_default_impl(extent_base_get(extent), extent_size_get(extent)); } else { extent_hook_pre_reentrancy(tsdn, arena); - err = (extent_hooks->dalloc == NULL || - extent_hooks->dalloc(extent_hooks, - extent_base_get(extent), extent_size_get(extent), - extent_committed_get(extent), arena_ind_get(arena))); + err = ehooks_dalloc(ehooks, extent_base_get(extent), + extent_size_get(extent), extent_committed_get(extent), + arena_ind_get(arena)); extent_hook_post_reentrancy(tsdn); } @@ -1491,51 +1477,48 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, } void -extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent) { +extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent) { assert(extent_dumpable_get(extent)); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); /* Avoid calling the default extent_dalloc unless have to. */ - if (extent_hooks != &extent_hooks_default || extent_may_dalloc()) { + if (!ehooks_are_default(ehooks) || extent_may_dalloc()) { /* * Deregister first to avoid a race with other allocating * threads, and reregister if deallocation fails. */ extent_deregister(tsdn, extent); - if (!extent_dalloc_wrapper_try(tsdn, arena, extent_hooks, - extent)) { + if (!extent_dalloc_wrapper_try(tsdn, arena, ehooks, extent)) { return; } extent_reregister(tsdn, extent); } - if (extent_hooks != &extent_hooks_default) { + if (!ehooks_are_default(ehooks)) { extent_hook_pre_reentrancy(tsdn, arena); } /* Try to decommit; purge if that fails. */ bool zeroed; if (!extent_committed_get(extent)) { zeroed = true; - } else if (!extent_decommit_wrapper(tsdn, arena, extent_hooks, extent, - 0, extent_size_get(extent))) { + } else if (!extent_decommit_wrapper(tsdn, arena, ehooks, extent, 0, + extent_size_get(extent))) { zeroed = true; - } else if (extent_hooks->purge_forced != NULL && - !extent_hooks->purge_forced(extent_hooks, - extent_base_get(extent), extent_size_get(extent), 0, - extent_size_get(extent), arena_ind_get(arena))) { + } else if (!ehooks_purge_forced(ehooks, extent_base_get(extent), + extent_size_get(extent), 0, extent_size_get(extent), + arena_ind_get(arena))) { zeroed = true; } else if (extent_state_get(extent) == extent_state_muzzy || - (extent_hooks->purge_lazy != NULL && - !extent_hooks->purge_lazy(extent_hooks, - extent_base_get(extent), extent_size_get(extent), 0, - extent_size_get(extent), arena_ind_get(arena)))) { + !ehooks_purge_lazy(ehooks, extent_base_get(extent), + extent_size_get(extent), 0, extent_size_get(extent), + arena_ind_get(arena))) { zeroed = false; } else { zeroed = false; } - if (extent_hooks != &extent_hooks_default) { + if (!ehooks_are_default(ehooks)) { extent_hook_post_reentrancy(tsdn); } extent_zeroed_set(extent, zeroed); @@ -1544,8 +1527,8 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_gdump_sub(tsdn, extent); } - extent_record(tsdn, arena, extent_hooks, &arena->eset_retained, - extent, false); + extent_record(tsdn, arena, ehooks, &arena->eset_retained, extent, + false); } static void @@ -1562,8 +1545,8 @@ extent_destroy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, } void -extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent) { +extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent) { assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -1575,15 +1558,15 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_addr_set(extent, extent_base_get(extent)); /* Try to destroy; silently fail otherwise. */ - if (extent_hooks == &extent_hooks_default) { + if (ehooks_are_default(ehooks)) { /* Call directly to propagate tsdn. */ extent_destroy_default_impl(extent_base_get(extent), extent_size_get(extent)); - } else if (extent_hooks->destroy != NULL) { + } else if (!ehooks_destroy_is_noop(ehooks)) { extent_hook_pre_reentrancy(tsdn, arena); - extent_hooks->destroy(extent_hooks, - extent_base_get(extent), extent_size_get(extent), - extent_committed_get(extent), arena_ind_get(arena)); + ehooks_destroy(ehooks, extent_base_get(extent), + extent_size_get(extent), extent_committed_get(extent), + arena_ind_get(arena)); extent_hook_post_reentrancy(tsdn); } @@ -1598,19 +1581,17 @@ extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, } static bool -extent_commit_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length, bool growing_retained) { +extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); - if (extent_hooks != &extent_hooks_default) { + if (!ehooks_are_default(ehooks)) { extent_hook_pre_reentrancy(tsdn, arena); } - bool err = (extent_hooks->commit == NULL || - extent_hooks->commit(extent_hooks, extent_base_get(extent), - extent_size_get(extent), offset, length, arena_ind_get(arena))); - if (extent_hooks != &extent_hooks_default) { + bool err = ehooks_commit(ehooks, extent_base_get(extent), + extent_size_get(extent), offset, length, arena_ind_get(arena)); + if (!ehooks_are_default(ehooks)) { extent_hook_post_reentrancy(tsdn); } extent_committed_set(extent, extent_committed_get(extent) || !err); @@ -1618,11 +1599,11 @@ extent_commit_impl(tsdn_t *tsdn, arena_t *arena, } bool -extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, +extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length) { - return extent_commit_impl(tsdn, arena, extent_hooks, extent, offset, - length, false); + return extent_commit_impl(tsdn, arena, ehooks, extent, offset, length, + false); } static bool @@ -1633,20 +1614,17 @@ extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, } bool -extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length) { +extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - if (extent_hooks != &extent_hooks_default) { + if (!ehooks_are_default(ehooks)) { extent_hook_pre_reentrancy(tsdn, arena); } - bool err = (extent_hooks->decommit == NULL || - extent_hooks->decommit(extent_hooks, - extent_base_get(extent), extent_size_get(extent), offset, length, - arena_ind_get(arena))); - if (extent_hooks != &extent_hooks_default) { + bool err = ehooks_decommit(ehooks, extent_base_get(extent), + extent_size_get(extent), offset, length, arena_ind_get(arena)); + if (!ehooks_are_default(ehooks)) { extent_hook_post_reentrancy(tsdn); } extent_committed_set(extent, extent_committed_get(extent) && err); @@ -1668,22 +1646,20 @@ extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, #endif static bool -extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length, bool growing_retained) { +extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); - if (extent_hooks->purge_lazy == NULL) { + if (ehooks_purge_lazy_will_fail(ehooks)) { return true; } - if (extent_hooks != &extent_hooks_default) { + if (!ehooks_are_default(ehooks)) { extent_hook_pre_reentrancy(tsdn, arena); } - bool err = extent_hooks->purge_lazy(extent_hooks, - extent_base_get(extent), extent_size_get(extent), offset, length, - arena_ind_get(arena)); - if (extent_hooks != &extent_hooks_default) { + bool err = ehooks_purge_lazy(ehooks, extent_base_get(extent), + extent_size_get(extent), offset, length, arena_ind_get(arena)); + if (!ehooks_are_default(ehooks)) { extent_hook_post_reentrancy(tsdn); } @@ -1691,11 +1667,10 @@ extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, } bool -extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length) { - return extent_purge_lazy_impl(tsdn, arena, extent_hooks, extent, - offset, length, false); +extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length) { + return extent_purge_lazy_impl(tsdn, arena, ehooks, extent, offset, + length, false); } #ifdef PAGES_CAN_PURGE_FORCED @@ -1713,32 +1688,29 @@ extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr, #endif static bool -extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length, bool growing_retained) { +extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); - if (extent_hooks->purge_forced == NULL) { + if (ehooks_purge_forced_will_fail(ehooks)) { return true; } - if (extent_hooks != &extent_hooks_default) { + if (!ehooks_are_default(ehooks)) { extent_hook_pre_reentrancy(tsdn, arena); } - bool err = extent_hooks->purge_forced(extent_hooks, - extent_base_get(extent), extent_size_get(extent), offset, length, - arena_ind_get(arena)); - if (extent_hooks != &extent_hooks_default) { + bool err = ehooks_purge_forced(ehooks, extent_base_get(extent), + extent_size_get(extent), offset, length, arena_ind_get(arena)); + if (!ehooks_are_default(ehooks)) { extent_hook_post_reentrancy(tsdn); } return err; } bool -extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, - size_t length) { - return extent_purge_forced_impl(tsdn, arena, extent_hooks, extent, +extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t offset, size_t length) { + return extent_purge_forced_impl(tsdn, arena, ehooks, extent, offset, length, false); } @@ -1765,15 +1737,14 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, * and returns the trail (except in case of error). */ static extent_t * -extent_split_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, - szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b, - bool growing_retained) { +extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a, + size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained) { assert(extent_size_get(extent) == size_a + size_b); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); - if (extent_hooks->split == NULL) { + if (ehooks_split_will_fail(ehooks)) { return NULL; } @@ -1815,13 +1786,13 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_lock2(tsdn, extent, trail); - if (extent_hooks != &extent_hooks_default) { + if (!ehooks_are_default(ehooks)) { extent_hook_pre_reentrancy(tsdn, arena); } - bool err = extent_hooks->split(extent_hooks, extent_base_get(extent), + bool err = ehooks_split(ehooks, extent_base_get(extent), size_a + size_b, size_a, size_b, extent_committed_get(extent), arena_ind_get(arena)); - if (extent_hooks != &extent_hooks_default) { + if (!ehooks_are_default(ehooks)) { extent_hook_post_reentrancy(tsdn); } if (err) { @@ -1848,11 +1819,11 @@ label_error_a: } extent_t * -extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, - szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) { - return extent_split_impl(tsdn, arena, extent_hooks, extent, size_a, - szind_a, slab_a, size_b, szind_b, slab_b, false); +extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a, + size_t size_b, szind_t szind_b, bool slab_b) { + return extent_split_impl(tsdn, arena, ehooks, extent, size_a, szind_a, + slab_a, size_b, szind_b, slab_b, false); } static bool @@ -1914,28 +1885,26 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, } static bool -extent_merge_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *a, extent_t *b, - bool growing_retained) { +extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a, + extent_t *b, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); assert(extent_base_get(a) < extent_base_get(b)); - if (extent_hooks->merge == NULL || extent_head_no_merge(a, b)) { + if (ehooks_merge_will_fail(ehooks) || extent_head_no_merge(a, b)) { return true; } bool err; - if (extent_hooks == &extent_hooks_default) { + if (ehooks_are_default(ehooks)) { /* Call directly to propagate tsdn. */ err = extent_merge_default_impl(extent_base_get(a), extent_base_get(b)); } else { extent_hook_pre_reentrancy(tsdn, arena); - err = extent_hooks->merge(extent_hooks, - extent_base_get(a), extent_size_get(a), extent_base_get(b), - extent_size_get(b), extent_committed_get(a), - arena_ind_get(arena)); + err = ehooks_merge(ehooks, extent_base_get(a), + extent_size_get(a), extent_base_get(b), extent_size_get(b), + extent_committed_get(a), arena_ind_get(arena)); extent_hook_post_reentrancy(tsdn); } @@ -1991,9 +1960,9 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, } bool -extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *a, extent_t *b) { - return extent_merge_impl(tsdn, arena, extent_hooks, a, b, false); +extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, + extent_t *a, extent_t *b) { + return extent_merge_impl(tsdn, arena, ehooks, a, b, false); } bool diff --git a/src/extent_dss.c b/src/extent_dss.c index dd80a196..59e7e7d6 100644 --- a/src/extent_dss.c +++ b/src/extent_dss.c @@ -194,9 +194,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, *commit = pages_decommit(ret, size); } if (*zero && *commit) { - extent_hooks_t *extent_hooks = - arena_get_extent_hooks(arena); extent_t extent; + ehooks_t *ehooks = arena_get_ehooks( + arena); extent_init(&extent, arena_ind_get(arena), ret, size, @@ -204,8 +204,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, extent_state_active, false, true, true, EXTENT_NOT_HEAD); if (extent_purge_forced_wrapper(tsdn, - arena, extent_hooks, &extent, 0, - size)) { + arena, ehooks, &extent, 0, size)) { memset(ret, 0, size); } } diff --git a/src/large.c b/src/large.c index 6de1c570..6fd21bea 100644 --- a/src/large.c +++ b/src/large.c @@ -93,20 +93,20 @@ static bool large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { arena_t *arena = arena_get_from_extent(extent); size_t oldusize = extent_usize_get(extent); - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); + ehooks_t *ehooks = arena_get_ehooks(arena); size_t diff = extent_size_get(extent) - (usize + sz_large_pad); assert(oldusize > usize); - if (extent_hooks->split == NULL) { + if (ehooks_split_will_fail(ehooks)) { return true; } /* Split excess pages. */ if (diff != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, - extent_hooks, extent, usize + sz_large_pad, - sz_size2index(usize), false, diff, SC_NSIZES, false); + ehooks, extent, usize + sz_large_pad, sz_size2index(usize), + false, diff, SC_NSIZES, false); if (trail == NULL) { return true; } @@ -116,7 +116,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { extent_size_get(trail)); } - arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, trail); + arena_extents_dirty_dalloc(tsdn, arena, ehooks, trail); } arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize); @@ -129,10 +129,10 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, bool zero) { arena_t *arena = arena_get_from_extent(extent); size_t oldusize = extent_usize_get(extent); - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); + ehooks_t *ehooks = arena_get_ehooks(arena); size_t trailsize = usize - oldusize; - if (extent_hooks->merge == NULL) { + if (ehooks_merge_will_fail(ehooks)) { return true; } @@ -149,17 +149,17 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, bool commit = true; extent_t *trail; bool new_mapping; - if ((trail = extents_alloc(tsdn, arena, extent_hooks, - &arena->eset_dirty, extent_past_get(extent), trailsize, 0, - CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL - || (trail = extents_alloc(tsdn, arena, extent_hooks, - &arena->eset_muzzy, extent_past_get(extent), trailsize, 0, - CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) { + if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty, + extent_past_get(extent), trailsize, 0, CACHELINE, false, SC_NSIZES, + &is_zeroed_trail, &commit)) != NULL + || (trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy, + extent_past_get(extent), trailsize, 0, CACHELINE, false, SC_NSIZES, + &is_zeroed_trail, &commit)) != NULL) { if (config_stats) { new_mapping = false; } } else { - if ((trail = extent_alloc_wrapper(tsdn, arena, extent_hooks, + if ((trail = extent_alloc_wrapper(tsdn, arena, ehooks, extent_past_get(extent), trailsize, 0, CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) { return true; @@ -169,8 +169,8 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, } } - if (extent_merge_wrapper(tsdn, arena, extent_hooks, extent, trail)) { - extent_dalloc_wrapper(tsdn, arena, extent_hooks, trail); + if (extent_merge_wrapper(tsdn, arena, ehooks, extent, trail)) { + extent_dalloc_wrapper(tsdn, arena, ehooks, trail); return true; } rtree_ctx_t rtree_ctx_fallback; @@ -339,8 +339,8 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent, static void large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { - extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); - arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, extent); + ehooks_t *ehooks = arena_get_ehooks(arena); + arena_extents_dirty_dalloc(tsdn, arena, ehooks, extent); } void