project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move extent ehook calls into ehooks

Browse Source
This commit is contained in:
David Goldblatt 2019-12-02 14:19:22 -08:00 committed by David Goldblatt
parent ba8b9ecbcb
commit ae0d8e8591
13 changed files with 511 additions and 404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/jemalloc/internal/arena_externs.h
View File

@ -28,7 +28,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
bin_stats_data_t *bstats, arena_stats_large_t *lstats, bin_stats_data_t *bstats, arena_stats_large_t *lstats,
arena_stats_extents_t *estats); arena_stats_extents_t *estats);
void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent); ehooks_t *ehooks, extent_t *extent);
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr); size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr);
#endif #endif
@ -72,7 +72,7 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, tcache_t *tcache, size_t size, size_t alignment, bool zero, tcache_t *tcache,
hook_ralloc_args_t *hook_args); hook_ralloc_args_t *hook_args);
dss_prec_t arena_dss_prec_get(arena_t *arena); dss_prec_t arena_dss_prec_get(arena_t *arena);
extent_hooks_t *arena_get_extent_hooks(arena_t *arena); ehooks_t *arena_get_ehooks(arena_t *arena);
extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena, extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena,
extent_hooks_t *extent_hooks); extent_hooks_t *extent_hooks);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);

9
include/jemalloc/internal/arena_inlines_b.h
View File

@ -14,11 +14,6 @@ arena_get_from_extent(extent_t *extent) {
ATOMIC_RELAXED); ATOMIC_RELAXED);
} }
JEMALLOC_ALWAYS_INLINE bool
arena_has_default_hooks(arena_t *arena) {
return (arena_get_extent_hooks(arena) == &extent_hooks_default);
}
JEMALLOC_ALWAYS_INLINE arena_t * JEMALLOC_ALWAYS_INLINE arena_t *
arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
if (arena != NULL) { if (arena != NULL) {
@ -134,10 +129,10 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
/* Purge a single extent to retained / unmapped directly. */ /* Purge a single extent to retained / unmapped directly. */
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t *extent_hooks, arena_decay_extent(tsdn_t *tsdn,arena_t *arena, ehooks_t *ehooks,
extent_t *extent) { extent_t *extent) {
size_t extent_size = extent_size_get(extent); size_t extent_size = extent_size_get(extent);
extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent); extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
if (config_stats) { if (config_stats) {
/* Update stats accordingly. */ /* Update stats accordingly. */
arena_stats_lock(tsdn, &arena->stats); arena_stats_lock(tsdn, &arena->stats);

2
include/jemalloc/internal/base_externs.h
View File

@ -7,7 +7,7 @@ extern const char *metadata_thp_mode_names[];
base_t *b0get(void); base_t *b0get(void);
base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
void base_delete(tsdn_t *tsdn, base_t *base); void base_delete(tsdn_t *tsdn, base_t *base);
extent_hooks_t *base_extent_hooks_get(base_t *base); ehooks_t *base_ehooks_get(base_t *base);
extent_hooks_t *base_extent_hooks_set(base_t *base, extent_hooks_t *base_extent_hooks_set(base_t *base,
extent_hooks_t *extent_hooks); extent_hooks_t *extent_hooks);
void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment); void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);

6
include/jemalloc/internal/base_structs.h
View File

@ -1,6 +1,7 @@
#ifndef JEMALLOC_INTERNAL_BASE_STRUCTS_H #ifndef JEMALLOC_INTERNAL_BASE_STRUCTS_H
#define JEMALLOC_INTERNAL_BASE_STRUCTS_H #define JEMALLOC_INTERNAL_BASE_STRUCTS_H
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/extent.h" #include "jemalloc/internal/extent.h"
#include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
@ -23,10 +24,9 @@ struct base_s {
unsigned ind; unsigned ind;
/* /*
* User-configurable extent hook functions. Points to an * User-configurable extent hook functions.
* extent_hooks_t.
*/ */
atomic_p_t extent_hooks; ehooks_t ehooks;
/* Protects base_alloc() and base_stats_get() operations. */ /* Protects base_alloc() and base_stats_get() operations. */
malloc_mutex_t mtx; malloc_mutex_t mtx;

147
include/jemalloc/internal/ehooks.h
View File

@ -1,4 +1,151 @@
#ifndef JEMALLOC_INTERNAL_EHOOKS_H #ifndef JEMALLOC_INTERNAL_EHOOKS_H
#define JEMALLOC_INTERNAL_EHOOKS_H #define JEMALLOC_INTERNAL_EHOOKS_H
#include "jemalloc/internal/atomic.h"
extern const extent_hooks_t extent_hooks_default;
typedef struct ehooks_s ehooks_t;
struct ehooks_s {
/* Logically an extent_hooks_t *. */
atomic_p_t ptr;
};
void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks);
static inline void
ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
atomic_store_p(&ehooks->ptr, extent_hooks, ATOMIC_RELEASE);
}
static inline extent_hooks_t *
ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE);
}
static inline bool
ehooks_are_default(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks) == &extent_hooks_default;
}
static inline bool
ehooks_destroy_is_noop(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->destroy == NULL;
}
static inline bool
ehooks_purge_lazy_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->purge_lazy == NULL;
}
static inline bool
ehooks_purge_forced_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->purge_forced == NULL;
}
static inline bool
ehooks_split_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL;
}
static inline bool
ehooks_merge_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
}
static inline void *
ehooks_alloc(ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool *commit, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
return extent_hooks->alloc(extent_hooks, new_addr, size, alignment,
zero, commit, arena_ind);
}
static inline bool
ehooks_dalloc(ehooks_t *ehooks, void *addr, size_t size, bool committed,
unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->dalloc == NULL) {
return true;
}
return extent_hooks->dalloc(extent_hooks, addr, size, committed,
arena_ind);
}
static inline void
ehooks_destroy(ehooks_t *ehooks, void *addr, size_t size, bool committed,
unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->destroy == NULL) {
return;
}
extent_hooks->destroy(extent_hooks, addr, size, committed, arena_ind);
}
static inline bool
ehooks_commit(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->commit == NULL) {
return true;
}
return extent_hooks->commit(extent_hooks, addr, size, offset, length,
arena_ind);
}
static inline bool
ehooks_decommit(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->decommit == NULL) {
return true;
}
return extent_hooks->decommit(extent_hooks, addr, size, offset, length,
arena_ind);
}
static inline bool
ehooks_purge_lazy(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->purge_lazy == NULL) {
return true;
}
return extent_hooks->purge_lazy(extent_hooks, addr, size, offset,
length, arena_ind);
}
static inline bool
ehooks_purge_forced(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->purge_forced == NULL) {
return true;
}
return extent_hooks->purge_forced(extent_hooks, addr, size, offset,
length, arena_ind);
}
static inline bool
ehooks_split(ehooks_t *ehooks, void *addr, size_t size, size_t size_a,
size_t size_b, bool committed, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->split == NULL) {
return true;
}
return extent_hooks->split(extent_hooks, addr, size, size_a, size_b,
committed, arena_ind);
}
static inline bool
ehooks_merge(ehooks_t *ehooks, void *addr_a, size_t size_a, void *addr_b,
size_t size_b, bool committed, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->merge == NULL) {
return true;
}
return extent_hooks->merge(extent_hooks, addr_a, size_a, addr_b, size_b,
committed, arena_ind);
}
#endif /* JEMALLOC_INTERNAL_EHOOKS_H */ #endif /* JEMALLOC_INTERNAL_EHOOKS_H */

61
include/jemalloc/internal/extent_externs.h
View File

@ -1,6 +1,7 @@
#ifndef JEMALLOC_INTERNAL_EXTENT_EXTERNS_H #ifndef JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
#define JEMALLOC_INTERNAL_EXTENT_EXTERNS_H #define JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_pool.h" #include "jemalloc/internal/mutex_pool.h"
#include "jemalloc/internal/ph.h" #include "jemalloc/internal/ph.h"
@ -9,7 +10,6 @@
extern size_t opt_lg_extent_max_active_fit; extern size_t opt_lg_extent_max_active_fit;
extern rtree_t extents_rtree; extern rtree_t extents_rtree;
extern const extent_hooks_t extent_hooks_default;
extern mutex_pool_t extent_mutex_pool; extern mutex_pool_t extent_mutex_pool;
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
@ -18,39 +18,34 @@ void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
ph_proto(, extent_avail_, extent_tree_t, extent_t) ph_proto(, extent_avail_, extent_tree_t, extent_t)
ph_proto(, extent_heap_, extent_heap_t, extent_t) ph_proto(, extent_heap_, extent_heap_t, extent_t)
extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, eset_t *eset, void *new_addr, eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment,
size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, bool slab, szind_t szind, bool *zero, bool *commit);
bool *zero, bool *commit); void extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void extents_dalloc(tsdn_t *tsdn, arena_t *arena, eset_t *eset, extent_t *extent);
extent_hooks_t *extent_hooks, eset_t *eset, extent_t *extent); extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, eset_t *eset, size_t npages_min);
extent_hooks_t *extent_hooks, eset_t *eset, size_t npages_min); extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad, szind_t szind, bool *zero, bool *commit);
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit);
void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent); void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent); extent_t *extent);
void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent); extent_t *extent);
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length);
size_t length); bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t offset, size_t length);
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
size_t length); extent_t *extent, size_t offset, size_t length);
bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length);
size_t length); extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, size_t size_b, szind_t szind_b, bool slab_b);
size_t length); bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *a, extent_t *b);
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b);
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b);
bool extent_boot(void); bool extent_boot(void);

93
src/arena.c
View File

@ -4,6 +4,7 @@
#include "jemalloc/internal/assert.h" #include "jemalloc/internal/assert.h"
#include "jemalloc/internal/div.h" #include "jemalloc/internal/div.h"
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
@ -252,13 +253,12 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
} }
void void
arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent) { extent_t *extent) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
extents_dalloc(tsdn, arena, extent_hooks, &arena->eset_dirty, extents_dalloc(tsdn, arena, ehooks, &arena->eset_dirty, extent);
extent);
if (arena_dirty_decay_ms_get(arena) == 0) { if (arena_dirty_decay_ms_get(arena) == 0) {
arena_decay_dirty(tsdn, arena, false, true); arena_decay_dirty(tsdn, arena, false, true);
} else { } else {
@ -426,7 +426,7 @@ arena_may_have_muzzy(arena_t *arena) {
extent_t * extent_t *
arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero) { size_t alignment, bool *zero) {
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
@ -434,19 +434,18 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
szind_t szind = sz_size2index(usize); szind_t szind = sz_size2index(usize);
size_t mapped_add; size_t mapped_add;
bool commit = true; bool commit = true;
extent_t *extent = extents_alloc(tsdn, arena, extent_hooks, extent_t *extent = extents_alloc(tsdn, arena, ehooks,
&arena->eset_dirty, NULL, usize, sz_large_pad, alignment, false, &arena->eset_dirty, NULL, usize, sz_large_pad, alignment, false,
szind, zero, &commit); szind, zero, &commit);
if (extent == NULL && arena_may_have_muzzy(arena)) { if (extent == NULL && arena_may_have_muzzy(arena)) {
extent = extents_alloc(tsdn, arena, extent_hooks, extent = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
&arena->eset_muzzy, NULL, usize, sz_large_pad, alignment, NULL, usize, sz_large_pad, alignment, false, szind, zero,
false, szind, zero, &commit); &commit);
} }
size_t size = usize + sz_large_pad; size_t size = usize + sz_large_pad;
if (extent == NULL) { if (extent == NULL) {
extent = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, extent = extent_alloc_wrapper(tsdn, arena, ehooks, NULL, usize,
usize, sz_large_pad, alignment, false, szind, zero, sz_large_pad, alignment, false, szind, zero, &commit);
&commit);
if (config_stats) { if (config_stats) {
/* /*
* extent may be NULL on OOM, but in that case * extent may be NULL on OOM, but in that case
@ -819,8 +818,8 @@ arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena,
static size_t static size_t
arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, eset_t *eset, size_t npages_limit, ehooks_t *ehooks, eset_t *eset, size_t npages_limit,
size_t npages_decay_max, extent_list_t *decay_extents) { size_t npages_decay_max, extent_list_t *decay_extents) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
@ -828,8 +827,8 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
size_t nstashed = 0; size_t nstashed = 0;
extent_t *extent; extent_t *extent;
while (nstashed < npages_decay_max && while (nstashed < npages_decay_max &&
(extent = extents_evict(tsdn, arena, extent_hooks, eset, (extent = extents_evict(tsdn, arena, ehooks, eset, npages_limit))
npages_limit)) != NULL) { != NULL) {
extent_list_append(decay_extents, extent); extent_list_append(decay_extents, extent);
nstashed += extent_size_get(extent) >> LG_PAGE; nstashed += extent_size_get(extent) >> LG_PAGE;
} }
@ -837,9 +836,9 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
} }
static size_t static size_t
arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, arena_decay_t *decay, eset_t *eset, arena_decay_t *decay, eset_t *eset, bool all, extent_list_t *decay_extents,
bool all, extent_list_t *decay_extents, bool is_background_thread) { bool is_background_thread) {
size_t nmadvise, nunmapped; size_t nmadvise, nunmapped;
size_t npurged; size_t npurged;
@ -864,9 +863,9 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena,
case extent_state_dirty: case extent_state_dirty:
if (!all && muzzy_decay_ms != 0 && if (!all && muzzy_decay_ms != 0 &&
!extent_purge_lazy_wrapper(tsdn, arena, !extent_purge_lazy_wrapper(tsdn, arena,
extent_hooks, extent, 0, ehooks, extent, 0,
extent_size_get(extent))) { extent_size_get(extent))) {
extents_dalloc(tsdn, arena, extent_hooks, extents_dalloc(tsdn, arena, ehooks,
&arena->eset_muzzy, extent); &arena->eset_muzzy, extent);
arena_background_thread_inactivity_check(tsdn, arena_background_thread_inactivity_check(tsdn,
arena, is_background_thread); arena, is_background_thread);
@ -874,8 +873,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena,
} }
JEMALLOC_FALLTHROUGH; JEMALLOC_FALLTHROUGH;
case extent_state_muzzy: case extent_state_muzzy:
extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
extent);
if (config_stats) { if (config_stats) {
nunmapped += npages; nunmapped += npages;
} }
@ -923,17 +921,16 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
decay->purging = true; decay->purging = true;
malloc_mutex_unlock(tsdn, &decay->mtx); malloc_mutex_unlock(tsdn, &decay->mtx);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
extent_list_t decay_extents; extent_list_t decay_extents;
extent_list_init(&decay_extents); extent_list_init(&decay_extents);
size_t npurge = arena_stash_decayed(tsdn, arena, extent_hooks, eset, size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, eset,
npages_limit, npages_decay_max, &decay_extents); npages_limit, npages_decay_max, &decay_extents);
if (npurge != 0) { if (npurge != 0) {
size_t npurged = arena_decay_stashed(tsdn, arena, size_t npurged = arena_decay_stashed(tsdn, arena, ehooks, decay,
extent_hooks, decay, eset, all, &decay_extents, eset, all, &decay_extents, is_background_thread);
is_background_thread);
assert(npurged == npurge); assert(npurged == npurge);
} }
@ -1006,8 +1003,8 @@ static void
arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) { arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) {
arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, slab); arena_extents_dirty_dalloc(tsdn, arena, ehooks, slab);
} }
static void static void
@ -1159,11 +1156,11 @@ arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) {
* destroyed, or provide custom extent hooks that track retained * destroyed, or provide custom extent hooks that track retained
* dss-based extents for later reuse. * dss-based extents for later reuse.
*/ */
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
extent_t *extent; extent_t *extent;
while ((extent = extents_evict(tsdn, arena, extent_hooks, while ((extent = extents_evict(tsdn, arena, ehooks,
&arena->eset_retained, 0)) != NULL) { &arena->eset_retained, 0)) != NULL) {
extent_destroy_wrapper(tsdn, arena, extent_hooks, extent); extent_destroy_wrapper(tsdn, arena, ehooks, extent);
} }
} }
@ -1204,9 +1201,8 @@ arena_destroy(tsd_t *tsd, arena_t *arena) {
} }
static extent_t * static extent_t *
arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, const bin_info_t *bin_info, const bin_info_t *bin_info, szind_t szind) {
szind_t szind) {
extent_t *slab; extent_t *slab;
bool zero, commit; bool zero, commit;
@ -1215,7 +1211,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
zero = false; zero = false;
commit = true; commit = true;
slab = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, slab = extent_alloc_wrapper(tsdn, arena, ehooks, NULL,
bin_info->slab_size, 0, PAGE, true, szind, &zero, &commit); bin_info->slab_size, 0, PAGE, true, szind, &zero, &commit);
if (config_stats && slab != NULL) { if (config_stats && slab != NULL) {
@ -1232,21 +1228,20 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
szind_t szind = sz_size2index(bin_info->reg_size); szind_t szind = sz_size2index(bin_info->reg_size);
bool zero = false; bool zero = false;
bool commit = true; bool commit = true;
extent_t *slab = extents_alloc(tsdn, arena, extent_hooks, extent_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
&arena->eset_dirty, NULL, bin_info->slab_size, 0, PAGE, true, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit);
binind, &zero, &commit);
if (slab == NULL && arena_may_have_muzzy(arena)) { if (slab == NULL && arena_may_have_muzzy(arena)) {
slab = extents_alloc(tsdn, arena, extent_hooks, slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
&arena->eset_muzzy, NULL, bin_info->slab_size, 0, PAGE, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero,
true, binind, &zero, &commit); &commit);
} }
if (slab == NULL) { if (slab == NULL) {
slab = arena_slab_alloc_hard(tsdn, arena, extent_hooks, slab = arena_slab_alloc_hard(tsdn, arena, ehooks, bin_info,
bin_info, szind); szind);
if (slab == NULL) { if (slab == NULL) {
return NULL; return NULL;
} }
@ -1846,9 +1841,9 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
return ret; return ret;
} }
extent_hooks_t * ehooks_t *
arena_get_extent_hooks(arena_t *arena) { arena_get_ehooks(arena_t *arena) {
return base_extent_hooks_get(arena->base); return base_ehooks_get(arena->base);
} }
extent_hooks_t * extent_hooks_t *

69
src/base.c
View File

@ -29,7 +29,7 @@ metadata_thp_madvise(void) {
} }
static void * static void *
base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) { base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) {
void *addr; void *addr;
bool zero = true; bool zero = true;
bool commit = true; bool commit = true;
@ -37,7 +37,7 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size)
/* Use huge page sizes and alignment regardless of opt_metadata_thp. */ /* Use huge page sizes and alignment regardless of opt_metadata_thp. */
assert(size == HUGEPAGE_CEILING(size)); assert(size == HUGEPAGE_CEILING(size));
size_t alignment = HUGEPAGE; size_t alignment = HUGEPAGE;
if (extent_hooks == &extent_hooks_default) { if (ehooks_are_default(ehooks)) {
addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit); addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit);
if (have_madvise_huge && addr) { if (have_madvise_huge && addr) {
pages_set_thp_state(addr, size); pages_set_thp_state(addr, size);
@ -46,8 +46,8 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size)
/* No arena context as we are creating new arenas. */ /* No arena context as we are creating new arenas. */
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
pre_reentrancy(tsd, NULL); pre_reentrancy(tsd, NULL);
addr = extent_hooks->alloc(extent_hooks, NULL, size, alignment, addr = ehooks_alloc(ehooks, NULL, size, alignment, &zero,
&zero, &commit, ind); &commit, ind);
post_reentrancy(tsd); post_reentrancy(tsd);
} }
@ -55,7 +55,7 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size)
} }
static void static void
base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr, base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr,
size_t size) { size_t size) {
/* /*
* Cascade through dalloc, decommit, purge_forced, and purge_lazy, * Cascade through dalloc, decommit, purge_forced, and purge_lazy,
@ -67,7 +67,7 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
* may in fact want the end state of all associated virtual memory to be * may in fact want the end state of all associated virtual memory to be
* in some consistent-but-allocated state. * in some consistent-but-allocated state.
*/ */
if (extent_hooks == &extent_hooks_default) { if (ehooks_are_default(ehooks)) {
if (!extent_dalloc_mmap(addr, size)) { if (!extent_dalloc_mmap(addr, size)) {
goto label_done; goto label_done;
} }
@ -85,24 +85,16 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
} else { } else {
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
pre_reentrancy(tsd, NULL); pre_reentrancy(tsd, NULL);
if (extent_hooks->dalloc != NULL && if (!ehooks_dalloc(ehooks, addr, size, true, ind)) {
!extent_hooks->dalloc(extent_hooks, addr, size, true,
ind)) {
goto label_post_reentrancy; goto label_post_reentrancy;
} }
if (extent_hooks->decommit != NULL && if (!ehooks_decommit(ehooks, addr, size, 0, size, ind)) {
!extent_hooks->decommit(extent_hooks, addr, size, 0, size,
ind)) {
goto label_post_reentrancy; goto label_post_reentrancy;
} }
if (extent_hooks->purge_forced != NULL && if (!ehooks_purge_forced(ehooks, addr, size, 0, size, ind)) {
!extent_hooks->purge_forced(extent_hooks, addr, size, 0,
size, ind)) {
goto label_post_reentrancy; goto label_post_reentrancy;
} }
if (extent_hooks->purge_lazy != NULL && if (!ehooks_purge_lazy(ehooks, addr, size, 0, size, ind)) {
!extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size,
ind)) {
goto label_post_reentrancy; goto label_post_reentrancy;
} }
/* Nothing worked. That's the application's problem. */ /* Nothing worked. That's the application's problem. */
@ -248,8 +240,8 @@ base_extent_bump_alloc(base_t *base, extent_t *extent, size_t size,
* On success a pointer to the initialized base_block_t header is returned. * On success a pointer to the initialized base_block_t header is returned.
*/ */
static base_block_t * static base_block_t *
base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks, base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind,
unsigned ind, pszind_t *pind_last, size_t *extent_sn_next, size_t size, pszind_t *pind_last, size_t *extent_sn_next, size_t size,
size_t alignment) { size_t alignment) {
alignment = ALIGNMENT_CEILING(alignment, QUANTUM); alignment = ALIGNMENT_CEILING(alignment, QUANTUM);
size_t usize = ALIGNMENT_CEILING(size, alignment); size_t usize = ALIGNMENT_CEILING(size, alignment);
@ -270,7 +262,7 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next)); size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next));
size_t block_size = (min_block_size > next_block_size) ? min_block_size size_t block_size = (min_block_size > next_block_size) ? min_block_size
: next_block_size; : next_block_size;
base_block_t *block = (base_block_t *)base_map(tsdn, extent_hooks, ind, base_block_t *block = (base_block_t *)base_map(tsdn, ehooks, ind,
block_size); block_size);
if (block == NULL) { if (block == NULL) {
return NULL; return NULL;
@ -311,13 +303,13 @@ static extent_t *
base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) { base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
malloc_mutex_assert_owner(tsdn, &base->mtx); malloc_mutex_assert_owner(tsdn, &base->mtx);
extent_hooks_t *extent_hooks = base_extent_hooks_get(base); ehooks_t *ehooks = base_ehooks_get(base);
/* /*
* Drop mutex during base_block_alloc(), because an extent hook will be * Drop mutex during base_block_alloc(), because an extent hook will be
* called. * called.
*/ */
malloc_mutex_unlock(tsdn, &base->mtx); malloc_mutex_unlock(tsdn, &base->mtx);
base_block_t *block = base_block_alloc(tsdn, base, extent_hooks, base_block_t *block = base_block_alloc(tsdn, base, ehooks,
base_ind_get(base), &base->pind_last, &base->extent_sn_next, size, base_ind_get(base), &base->pind_last, &base->extent_sn_next, size,
alignment); alignment);
malloc_mutex_lock(tsdn, &base->mtx); malloc_mutex_lock(tsdn, &base->mtx);
@ -353,7 +345,16 @@ base_t *
base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
pszind_t pind_last = 0; pszind_t pind_last = 0;
size_t extent_sn_next = 0; size_t extent_sn_next = 0;
base_block_t *block = base_block_alloc(tsdn, NULL, extent_hooks, ind,
/*
* The base will contain the ehooks eventually, but it itself is
* allocated using them. So we use some stack ehooks to bootstrap its
* memory, and then initialize the ehooks within the base_t.
*/
ehooks_t fake_ehooks;
ehooks_init(&fake_ehooks, extent_hooks);
base_block_t *block = base_block_alloc(tsdn, NULL, &fake_ehooks, ind,
&pind_last, &extent_sn_next, sizeof(base_t), QUANTUM); &pind_last, &extent_sn_next, sizeof(base_t), QUANTUM);
if (block == NULL) { if (block == NULL) {
return NULL; return NULL;
@ -365,10 +366,10 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
base_t *base = (base_t *)base_extent_bump_alloc_helper(&block->extent, base_t *base = (base_t *)base_extent_bump_alloc_helper(&block->extent,
&gap_size, base_size, base_alignment); &gap_size, base_size, base_alignment);
base->ind = ind; base->ind = ind;
atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELAXED); ehooks_init(&base->ehooks, extent_hooks);
if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE, if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE,
malloc_mutex_rank_exclusive)) { malloc_mutex_rank_exclusive)) {
base_unmap(tsdn, extent_hooks, ind, block, block->size); base_unmap(tsdn, &fake_ehooks, ind, block, block->size);
return NULL; return NULL;
} }
base->pind_last = pind_last; base->pind_last = pind_last;
@ -397,26 +398,26 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
void void
base_delete(tsdn_t *tsdn, base_t *base) { base_delete(tsdn_t *tsdn, base_t *base) {
extent_hooks_t *extent_hooks = base_extent_hooks_get(base); ehooks_t *ehooks = base_ehooks_get(base);
base_block_t *next = base->blocks; base_block_t *next = base->blocks;
do { do {
base_block_t *block = next; base_block_t *block = next;
next = block->next; next = block->next;
base_unmap(tsdn, extent_hooks, base_ind_get(base), block, base_unmap(tsdn, ehooks, base_ind_get(base), block,
block->size); block->size);
} while (next != NULL); } while (next != NULL);
} }
extent_hooks_t * ehooks_t *
base_extent_hooks_get(base_t *base) { base_ehooks_get(base_t *base) {
return (extent_hooks_t *)atomic_load_p(&base->extent_hooks, return &base->ehooks;
ATOMIC_ACQUIRE);
} }
extent_hooks_t * extent_hooks_t *
base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks) { base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks) {
extent_hooks_t *old_extent_hooks = base_extent_hooks_get(base); extent_hooks_t *old_extent_hooks =
atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELEASE); ehooks_get_extent_hooks_ptr(&base->ehooks);
ehooks_init(&base->ehooks, extent_hooks);
return old_extent_hooks; return old_extent_hooks;
} }

5
src/ctl.c
View File

@ -2400,8 +2400,9 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
arena, new_extent_hooks); arena, new_extent_hooks);
READ(old_extent_hooks, extent_hooks_t *); READ(old_extent_hooks, extent_hooks_t *);
} else { } else {
old_extent_hooks = arena_get_extent_hooks( old_extent_hooks =
arena); ehooks_get_extent_hooks_ptr(
arena_get_ehooks(arena));
READ(old_extent_hooks, extent_hooks_t *); READ(old_extent_hooks, extent_hooks_t *);
} }
} }

5
src/ehooks.c
View File

@ -1,3 +1,8 @@
#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/ehooks.h"
void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
ehooks_set_extent_hooks_ptr(ehooks, extent_hooks);
}

471
src/extent.c
View File

@ -28,40 +28,38 @@ static void extent_destroy_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, bool committed, unsigned arena_ind); size_t size, bool committed, unsigned arena_ind);
static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr, static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind); size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length, bool growing_retained);
size_t length, bool growing_retained); static bool extent_decommit_default(extent_hooks_t *extent_hooks, void *addr,
static bool extent_decommit_default(extent_hooks_t *extent_hooks, size_t size, size_t offset, size_t length, unsigned arena_ind);
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
#ifdef PAGES_CAN_PURGE_LAZY #ifdef PAGES_CAN_PURGE_LAZY
static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind); size_t size, size_t offset, size_t length, unsigned arena_ind);
#endif #endif
static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length,
size_t length, bool growing_retained); bool growing_retained);
#ifdef PAGES_CAN_PURGE_FORCED #ifdef PAGES_CAN_PURGE_FORCED
static bool extent_purge_forced_default(extent_hooks_t *extent_hooks, static bool extent_purge_forced_default(extent_hooks_t *extent_hooks,
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
#endif #endif
static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length,
size_t length, bool growing_retained); bool growing_retained);
static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t size_a, size_t size_b, bool committed, size_t size, size_t size_a, size_t size_b, bool committed,
unsigned arena_ind); unsigned arena_ind);
static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena, static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, ehooks_t *ehooks, extent_t *extent, size_t size_a, szind_t szind_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
bool growing_retained); bool growing_retained);
static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a,
size_t size_a, void *addr_b, size_t size_b, bool committed, size_t size_a, void *addr_b, size_t size_b, bool committed,
unsigned arena_ind); unsigned arena_ind);
static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b, extent_t *a, extent_t *b, bool growing_retained);
bool growing_retained);
const extent_hooks_t extent_hooks_default = { const extent_hooks_t extent_hooks_default = {
extent_alloc_default, extent_alloc_default,
extent_dalloc_default, extent_dalloc_default,
extent_destroy_default, extent_destroy_default,
@ -97,16 +95,14 @@ static atomic_zu_t highpages;
*/ */
static void extent_deregister(tsdn_t *tsdn, extent_t *extent); static void extent_deregister(tsdn_t *tsdn, extent_t *extent);
static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, eset_t *eset, void *new_addr, eset_t *eset, void *new_addr, size_t usize, size_t pad, size_t alignment,
size_t usize, size_t pad, size_t alignment, bool slab, szind_t szind, bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained);
bool *zero, bool *commit, bool growing_retained);
static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent,
extent_t *extent, bool *coalesced, bool growing_retained); bool *coalesced, bool growing_retained);
static void extent_record(tsdn_t *tsdn, arena_t *arena, static void extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, eset_t *eset, extent_t *extent, eset_t *eset, extent_t *extent, bool growing_retained);
bool growing_retained);
/******************************************************************************/ /******************************************************************************/
@ -224,13 +220,12 @@ extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp) ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp)
static bool static bool
extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent) {
extent_t *extent) {
extent_state_set(extent, extent_state_active); extent_state_set(extent, extent_state_active);
bool coalesced; bool coalesced;
extent = extent_try_coalesce(tsdn, arena, extent_hooks, rtree_ctx, extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, eset,
eset, extent, &coalesced, false); extent, &coalesced, false);
extent_state_set(extent, eset_state_get(eset)); extent_state_set(extent, eset_state_get(eset));
if (!coalesced) { if (!coalesced) {
@ -241,23 +236,23 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena,
} }
extent_t * extent_t *
extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
eset_t *eset, void *new_addr, size_t size, size_t pad, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { szind_t szind, bool *zero, bool *commit) {
assert(size + pad != 0); assert(size + pad != 0);
assert(alignment != 0); assert(alignment != 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
extent_t *extent = extent_recycle(tsdn, arena, extent_hooks, eset, extent_t *extent = extent_recycle(tsdn, arena, ehooks, eset, new_addr,
new_addr, size, pad, alignment, slab, szind, zero, commit, false); size, pad, alignment, slab, szind, zero, commit, false);
assert(extent == NULL || extent_dumpable_get(extent)); assert(extent == NULL || extent_dumpable_get(extent));
return extent; return extent;
} }
void void
extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
eset_t *eset, extent_t *extent) { extent_t *extent) {
assert(extent_base_get(extent) != NULL); assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0); assert(extent_size_get(extent) != 0);
assert(extent_dumpable_get(extent)); assert(extent_dumpable_get(extent));
@ -267,12 +262,12 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
extent_zeroed_set(extent, false); extent_zeroed_set(extent, false);
extent_record(tsdn, arena, extent_hooks, eset, extent, false); extent_record(tsdn, arena, ehooks, eset, extent, false);
} }
extent_t * extent_t *
extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
eset_t *eset, size_t npages_min) { size_t npages_min) {
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
@ -301,8 +296,8 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
break; break;
} }
/* Try to coalesce. */ /* Try to coalesce. */
if (extent_try_delayed_coalesce(tsdn, arena, extent_hooks, if (extent_try_delayed_coalesce(tsdn, arena, ehooks, rtree_ctx,
rtree_ctx, eset, extent)) { eset, extent)) {
break; break;
} }
/* /*
@ -339,8 +334,8 @@ label_return:
* indicates OOM), e.g. when trying to split an existing extent. * indicates OOM), e.g. when trying to split an existing extent.
*/ */
static void static void
extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
eset_t *eset, extent_t *extent, bool growing_retained) { extent_t *extent, bool growing_retained) {
size_t sz = extent_size_get(extent); size_t sz = extent_size_get(extent);
if (config_stats) { if (config_stats) {
arena_stats_accum_zu(&arena->stats.abandoned_vm, sz); arena_stats_accum_zu(&arena->stats.abandoned_vm, sz);
@ -350,11 +345,10 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
* that this is only a virtual memory leak. * that this is only a virtual memory leak.
*/ */
if (eset_state_get(eset) == extent_state_dirty) { if (eset_state_get(eset) == extent_state_dirty) {
if (extent_purge_lazy_impl(tsdn, arena, extent_hooks, if (extent_purge_lazy_impl(tsdn, arena, ehooks, extent, 0, sz,
extent, 0, sz, growing_retained)) { growing_retained)) {
extent_purge_forced_impl(tsdn, arena, extent_hooks, extent_purge_forced_impl(tsdn, arena, ehooks, extent, 0,
extent, 0, extent_size_get(extent), extent_size_get(extent), growing_retained);
growing_retained);
} }
} }
extent_dalloc(tsdn, arena, extent); extent_dalloc(tsdn, arena, extent);
@ -581,10 +575,9 @@ extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) {
* given allocation request. * given allocation request.
*/ */
static extent_t * static extent_t *
extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, size_t pad, size_t alignment, bool slab, bool growing_retained) {
bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0); WITNESS_RANK_CORE, growing_retained ? 1 : 0);
assert(alignment > 0); assert(alignment > 0);
@ -668,8 +661,8 @@ typedef enum {
} extent_split_interior_result_t; } extent_split_interior_result_t;
static extent_split_interior_result_t static extent_split_interior_result_t
extent_split_interior(tsdn_t *tsdn, arena_t *arena, extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, rtree_ctx_t *rtree_ctx,
/* The result of splitting, in case of success. */ /* The result of splitting, in case of success. */
extent_t **extent, extent_t **lead, extent_t **trail, extent_t **extent, extent_t **lead, extent_t **trail,
/* The mess to clean up, in case of error. */ /* The mess to clean up, in case of error. */
@ -693,9 +686,9 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena,
/* Split the lead. */ /* Split the lead. */
if (leadsize != 0) { if (leadsize != 0) {
*lead = *extent; *lead = *extent;
*extent = extent_split_impl(tsdn, arena, extent_hooks, *extent = extent_split_impl(tsdn, arena, ehooks, *lead,
*lead, leadsize, SC_NSIZES, false, esize + trailsize, szind, leadsize, SC_NSIZES, false, esize + trailsize, szind, slab,
slab, growing_retained); growing_retained);
if (*extent == NULL) { if (*extent == NULL) {
*to_leak = *lead; *to_leak = *lead;
*lead = NULL; *lead = NULL;
@ -705,9 +698,8 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena,
/* Split the trail. */ /* Split the trail. */
if (trailsize != 0) { if (trailsize != 0) {
*trail = extent_split_impl(tsdn, arena, extent_hooks, *extent, *trail = extent_split_impl(tsdn, arena, ehooks, *extent, esize,
esize, szind, slab, trailsize, SC_NSIZES, false, szind, slab, trailsize, SC_NSIZES, false, growing_retained);
growing_retained);
if (*trail == NULL) { if (*trail == NULL) {
*to_leak = *extent; *to_leak = *extent;
*to_salvage = *lead; *to_salvage = *lead;
@ -745,18 +737,18 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena,
* and put back into eset. * and put back into eset.
*/ */
static extent_t * static extent_t *
extent_recycle_split(tsdn_t *tsdn, arena_t *arena, extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, size_t pad, size_t alignment, bool slab, szind_t szind, extent_t *extent,
szind_t szind, extent_t *extent, bool growing_retained) { bool growing_retained) {
extent_t *lead; extent_t *lead;
extent_t *trail; extent_t *trail;
extent_t *to_leak; extent_t *to_leak;
extent_t *to_salvage; extent_t *to_salvage;
extent_split_interior_result_t result = extent_split_interior( extent_split_interior_result_t result = extent_split_interior(
tsdn, arena, extent_hooks, rtree_ctx, &extent, &lead, &trail, tsdn, arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak,
&to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind, &to_salvage, new_addr, size, pad, alignment, slab, szind,
growing_retained); growing_retained);
if (!maps_coalesce && result != extent_split_interior_ok if (!maps_coalesce && result != extent_split_interior_ok
@ -790,8 +782,8 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena,
if (to_leak != NULL) { if (to_leak != NULL) {
void *leak = extent_base_get(to_leak); void *leak = extent_base_get(to_leak);
extent_deregister_no_gdump_sub(tsdn, to_leak); extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, extent_hooks, eset, extents_abandon_vm(tsdn, arena, ehooks, eset, to_leak,
to_leak, growing_retained); growing_retained);
assert(extent_lock_from_addr(tsdn, rtree_ctx, leak, assert(extent_lock_from_addr(tsdn, rtree_ctx, leak,
false) == NULL); false) == NULL);
} }
@ -807,7 +799,7 @@ extent_need_manual_zero(arena_t *arena) {
* default extent hooks installed (in which case the purge semantics may * default extent hooks installed (in which case the purge semantics may
* change); or 2) transparent huge pages enabled. * change); or 2) transparent huge pages enabled.
*/ */
return (!arena_has_default_hooks(arena) || return (!ehooks_are_default(arena_get_ehooks(arena)) ||
(opt_thp == thp_mode_always)); (opt_thp == thp_mode_always));
} }
@ -816,10 +808,9 @@ extent_need_manual_zero(arena_t *arena) {
* in the given eset_t. * in the given eset_t.
*/ */
static extent_t * static extent_t *
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
eset_t *eset, void *new_addr, size_t size, size_t pad, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit, szind_t szind, bool *zero, bool *commit, bool growing_retained) {
bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0); WITNESS_RANK_CORE, growing_retained ? 1 : 0);
assert(new_addr == NULL || !slab); assert(new_addr == NULL || !slab);
@ -829,25 +820,25 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
extent_t *extent = extent_recycle_extract(tsdn, arena, extent_hooks, extent_t *extent = extent_recycle_extract(tsdn, arena, ehooks,
rtree_ctx, eset, new_addr, size, pad, alignment, slab, rtree_ctx, eset, new_addr, size, pad, alignment, slab,
growing_retained); growing_retained);
if (extent == NULL) { if (extent == NULL) {
return NULL; return NULL;
} }
extent = extent_recycle_split(tsdn, arena, extent_hooks, rtree_ctx, extent = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, eset,
eset, new_addr, size, pad, alignment, slab, szind, extent, new_addr, size, pad, alignment, slab, szind, extent,
growing_retained); growing_retained);
if (extent == NULL) { if (extent == NULL) {
return NULL; return NULL;
} }
if (*commit && !extent_committed_get(extent)) { if (*commit && !extent_committed_get(extent)) {
if (extent_commit_impl(tsdn, arena, extent_hooks, extent, if (extent_commit_impl(tsdn, arena, ehooks, extent, 0,
0, extent_size_get(extent), growing_retained)) { extent_size_get(extent), growing_retained)) {
extent_record(tsdn, arena, extent_hooks, eset, extent_record(tsdn, arena, ehooks, eset, extent,
extent, growing_retained); growing_retained);
return NULL; return NULL;
} }
if (!extent_need_manual_zero(arena)) { if (!extent_need_manual_zero(arena)) {
@ -985,9 +976,9 @@ extent_hook_post_reentrancy(tsdn_t *tsdn) {
* virtual memory ranges retained by each arena. * virtual memory ranges retained by each arena.
*/ */
static extent_t * static extent_t *
extent_grow_retained(tsdn_t *tsdn, arena_t *arena, extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, size_t size, size_t pad, size_t alignment, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
bool slab, szind_t szind, bool *zero, bool *commit) { bool *zero, bool *commit) {
malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx); malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx);
assert(pad == 0 || !slab); assert(pad == 0 || !slab);
assert(!*zero || !slab); assert(!*zero || !slab);
@ -1022,14 +1013,13 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
bool committed = false; bool committed = false;
void *ptr; void *ptr;
if (extent_hooks == &extent_hooks_default) { if (ehooks_are_default(ehooks)) {
ptr = extent_alloc_default_impl(tsdn, arena, NULL, ptr = extent_alloc_default_impl(tsdn, arena, NULL,
alloc_size, PAGE, &zeroed, &committed); alloc_size, PAGE, &zeroed, &committed);
} else { } else {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
ptr = extent_hooks->alloc(extent_hooks, NULL, ptr = ehooks_alloc(ehooks, NULL, alloc_size, PAGE, &zeroed,
alloc_size, PAGE, &zeroed, &committed, &committed, arena_ind_get(arena));
arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
@ -1060,18 +1050,17 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
extent_t *trail; extent_t *trail;
extent_t *to_leak; extent_t *to_leak;
extent_t *to_salvage; extent_t *to_salvage;
extent_split_interior_result_t result = extent_split_interior( extent_split_interior_result_t result = extent_split_interior(tsdn,
tsdn, arena, extent_hooks, rtree_ctx, &extent, &lead, &trail, arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak,
&to_leak, &to_salvage, NULL, size, pad, alignment, slab, szind, &to_salvage, NULL, size, pad, alignment, slab, szind, true);
true);
if (result == extent_split_interior_ok) { if (result == extent_split_interior_ok) {
if (lead != NULL) { if (lead != NULL) {
extent_record(tsdn, arena, extent_hooks, extent_record(tsdn, arena, ehooks,
&arena->eset_retained, lead, true); &arena->eset_retained, lead, true);
} }
if (trail != NULL) { if (trail != NULL) {
extent_record(tsdn, arena, extent_hooks, extent_record(tsdn, arena, ehooks,
&arena->eset_retained, trail, true); &arena->eset_retained, trail, true);
} }
} else { } else {
@ -1084,21 +1073,21 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
if (config_prof) { if (config_prof) {
extent_gdump_add(tsdn, to_salvage); extent_gdump_add(tsdn, to_salvage);
} }
extent_record(tsdn, arena, extent_hooks, extent_record(tsdn, arena, ehooks,
&arena->eset_retained, to_salvage, true); &arena->eset_retained, to_salvage, true);
} }
if (to_leak != NULL) { if (to_leak != NULL) {
extent_deregister_no_gdump_sub(tsdn, to_leak); extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, extent_hooks, extents_abandon_vm(tsdn, arena, ehooks,
&arena->eset_retained, to_leak, true); &arena->eset_retained, to_leak, true);
} }
goto label_err; goto label_err;
} }
if (*commit && !extent_committed_get(extent)) { if (*commit && !extent_committed_get(extent)) {
if (extent_commit_impl(tsdn, arena, extent_hooks, extent, 0, if (extent_commit_impl(tsdn, arena, ehooks, extent, 0,
extent_size_get(extent), true)) { extent_size_get(extent), true)) {
extent_record(tsdn, arena, extent_hooks, extent_record(tsdn, arena, ehooks,
&arena->eset_retained, extent, true); &arena->eset_retained, extent, true);
goto label_err; goto label_err;
} }
@ -1151,15 +1140,15 @@ label_err:
} }
static extent_t * static extent_t *
extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { szind_t szind, bool *zero, bool *commit) {
assert(size != 0); assert(size != 0);
assert(alignment != 0); assert(alignment != 0);
malloc_mutex_lock(tsdn, &arena->extent_grow_mtx); malloc_mutex_lock(tsdn, &arena->extent_grow_mtx);
extent_t *extent = extent_recycle(tsdn, arena, extent_hooks, extent_t *extent = extent_recycle(tsdn, arena, ehooks,
&arena->eset_retained, new_addr, size, pad, alignment, slab, &arena->eset_retained, new_addr, size, pad, alignment, slab,
szind, zero, commit, true); szind, zero, commit, true);
if (extent != NULL) { if (extent != NULL) {
@ -1168,8 +1157,8 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
extent_gdump_add(tsdn, extent); extent_gdump_add(tsdn, extent);
} }
} else if (opt_retain && new_addr == NULL) { } else if (opt_retain && new_addr == NULL) {
extent = extent_grow_retained(tsdn, arena, extent_hooks, size, extent = extent_grow_retained(tsdn, arena, ehooks, size, pad,
pad, alignment, slab, szind, zero, commit); alignment, slab, szind, zero, commit);
/* extent_grow_retained() always releases extent_grow_mtx. */ /* extent_grow_retained() always releases extent_grow_mtx. */
} else { } else {
malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
@ -1180,9 +1169,9 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
} }
static extent_t * static extent_t *
extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { szind_t szind, bool *zero, bool *commit) {
size_t esize = size + pad; size_t esize = size + pad;
extent_t *extent = extent_alloc(tsdn, arena); extent_t *extent = extent_alloc(tsdn, arena);
if (extent == NULL) { if (extent == NULL) {
@ -1190,14 +1179,14 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
} }
void *addr; void *addr;
size_t palignment = ALIGNMENT_CEILING(alignment, PAGE); size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
if (extent_hooks == &extent_hooks_default) { if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */ /* Call directly to propagate tsdn. */
addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize, addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize,
palignment, zero, commit); palignment, zero, commit);
} else { } else {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
addr = extent_hooks->alloc(extent_hooks, new_addr, addr = ehooks_alloc(ehooks, new_addr, esize, palignment, zero,
esize, palignment, zero, commit, arena_ind_get(arena)); commit, arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
if (addr == NULL) { if (addr == NULL) {
@ -1219,14 +1208,14 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
} }
extent_t * extent_t *
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { szind_t szind, bool *zero, bool *commit) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
extent_t *extent = extent_alloc_retained(tsdn, arena, extent_hooks, extent_t *extent = extent_alloc_retained(tsdn, arena, ehooks, new_addr,
new_addr, size, pad, alignment, slab, szind, zero, commit); size, pad, alignment, slab, szind, zero, commit);
if (extent == NULL) { if (extent == NULL) {
if (opt_retain && new_addr != NULL) { if (opt_retain && new_addr != NULL) {
/* /*
@ -1237,7 +1226,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
*/ */
return NULL; return NULL;
} }
extent = extent_alloc_wrapper_hard(tsdn, arena, extent_hooks, extent = extent_alloc_wrapper_hard(tsdn, arena, ehooks,
new_addr, size, pad, alignment, slab, szind, zero, commit); new_addr, size, pad, alignment, slab, szind, zero, commit);
} }
@ -1266,15 +1255,14 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const extent_t *inner,
} }
static bool static bool
extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
eset_t *eset, extent_t *inner, extent_t *outer, bool forward, extent_t *inner, extent_t *outer, bool forward, bool growing_retained) {
bool growing_retained) {
assert(extent_can_coalesce(arena, eset, inner, outer)); assert(extent_can_coalesce(arena, eset, inner, outer));
extent_activate_locked(tsdn, arena, eset, outer); extent_activate_locked(tsdn, arena, eset, outer);
malloc_mutex_unlock(tsdn, &eset->mtx); malloc_mutex_unlock(tsdn, &eset->mtx);
bool err = extent_merge_impl(tsdn, arena, extent_hooks, bool err = extent_merge_impl(tsdn, arena, ehooks,
forward ? inner : outer, forward ? outer : inner, growing_retained); forward ? inner : outer, forward ? outer : inner, growing_retained);
malloc_mutex_lock(tsdn, &eset->mtx); malloc_mutex_lock(tsdn, &eset->mtx);
@ -1286,10 +1274,9 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
} }
static extent_t * static extent_t *
extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
extent_t *extent, bool *coalesced, bool growing_retained, bool growing_retained, bool inactive_only) {
bool inactive_only) {
/* /*
* We avoid checking / locking inactive neighbors for large size * We avoid checking / locking inactive neighbors for large size
* classes, since they are eagerly coalesced on deallocation which can * classes, since they are eagerly coalesced on deallocation which can
@ -1318,7 +1305,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena,
extent_unlock(tsdn, next); extent_unlock(tsdn, next);
if (can_coalesce && !extent_coalesce(tsdn, arena, if (can_coalesce && !extent_coalesce(tsdn, arena,
extent_hooks, eset, extent, next, true, ehooks, eset, extent, next, true,
growing_retained)) { growing_retained)) {
if (eset->delay_coalesce) { if (eset->delay_coalesce) {
/* Do minimal coalescing. */ /* Do minimal coalescing. */
@ -1338,7 +1325,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena,
extent_unlock(tsdn, prev); extent_unlock(tsdn, prev);
if (can_coalesce && !extent_coalesce(tsdn, arena, if (can_coalesce && !extent_coalesce(tsdn, arena,
extent_hooks, eset, extent, prev, false, ehooks, eset, extent, prev, false,
growing_retained)) { growing_retained)) {
extent = prev; extent = prev;
if (eset->delay_coalesce) { if (eset->delay_coalesce) {
@ -1358,19 +1345,19 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena,
} }
static extent_t * static extent_t *
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
extent_t *extent, bool *coalesced, bool growing_retained) { bool growing_retained) {
return extent_try_coalesce_impl(tsdn, arena, extent_hooks, rtree_ctx, return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
eset, extent, coalesced, growing_retained, false); extent, coalesced, growing_retained, false);
} }
static extent_t * static extent_t *
extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
extent_t *extent, bool *coalesced, bool growing_retained) { bool growing_retained) {
return extent_try_coalesce_impl(tsdn, arena, extent_hooks, rtree_ctx, return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
eset, extent, coalesced, growing_retained, true); extent, coalesced, growing_retained, true);
} }
/* /*
@ -1378,8 +1365,8 @@ extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena,
* given eset_t (coalesces, deregisters slab interiors, the heap operations). * given eset_t (coalesces, deregisters slab interiors, the heap operations).
*/ */
static void static void
extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
eset_t *eset, extent_t *extent, bool growing_retained) { extent_t *extent, bool growing_retained) {
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
@ -1399,22 +1386,22 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
(uintptr_t)extent_base_get(extent), true) == extent); (uintptr_t)extent_base_get(extent), true) == extent);
if (!eset->delay_coalesce) { if (!eset->delay_coalesce) {
extent = extent_try_coalesce(tsdn, arena, extent_hooks, extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx,
rtree_ctx, eset, extent, NULL, growing_retained); eset, extent, NULL, growing_retained);
} else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) { } else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) {
assert(eset == &arena->eset_dirty); assert(eset == &arena->eset_dirty);
/* Always coalesce large eset eagerly. */ /* Always coalesce large eset eagerly. */
bool coalesced; bool coalesced;
do { do {
assert(extent_state_get(extent) == extent_state_active); assert(extent_state_get(extent) == extent_state_active);
extent = extent_try_coalesce_large(tsdn, arena, extent = extent_try_coalesce_large(tsdn, arena, ehooks,
extent_hooks, rtree_ctx, eset, extent, rtree_ctx, eset, extent, &coalesced,
&coalesced, growing_retained); growing_retained);
} while (coalesced); } while (coalesced);
if (extent_size_get(extent) >= oversize_threshold) { if (extent_size_get(extent) >= oversize_threshold) {
/* Shortcut to purge the oversize extent eagerly. */ /* Shortcut to purge the oversize extent eagerly. */
malloc_mutex_unlock(tsdn, &eset->mtx); malloc_mutex_unlock(tsdn, &eset->mtx);
arena_decay_extent(tsdn, arena, extent_hooks, extent); arena_decay_extent(tsdn, arena, ehooks, extent);
return; return;
} }
} }
@ -1425,7 +1412,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
void void
extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
@ -1434,7 +1421,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_dalloc(tsdn, arena, extent); extent_dalloc(tsdn, arena, extent);
return; return;
} }
extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent); extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
} }
static bool static bool
@ -1458,8 +1445,8 @@ extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
} }
static bool static bool
extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent) { extent_t *extent) {
bool err; bool err;
assert(extent_base_get(extent) != NULL); assert(extent_base_get(extent) != NULL);
@ -1470,16 +1457,15 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
/* Try to deallocate. */ /* Try to deallocate. */
if (extent_hooks == &extent_hooks_default) { if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */ /* Call directly to propagate tsdn. */
err = extent_dalloc_default_impl(extent_base_get(extent), err = extent_dalloc_default_impl(extent_base_get(extent),
extent_size_get(extent)); extent_size_get(extent));
} else { } else {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
err = (extent_hooks->dalloc == NULL || err = ehooks_dalloc(ehooks, extent_base_get(extent),
extent_hooks->dalloc(extent_hooks, extent_size_get(extent), extent_committed_get(extent),
extent_base_get(extent), extent_size_get(extent), arena_ind_get(arena));
extent_committed_get(extent), arena_ind_get(arena)));
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
@ -1491,51 +1477,48 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
} }
void void
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent) { extent_t *extent) {
assert(extent_dumpable_get(extent)); assert(extent_dumpable_get(extent));
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
/* Avoid calling the default extent_dalloc unless have to. */ /* Avoid calling the default extent_dalloc unless have to. */
if (extent_hooks != &extent_hooks_default || extent_may_dalloc()) { if (!ehooks_are_default(ehooks) || extent_may_dalloc()) {
/* /*
* Deregister first to avoid a race with other allocating * Deregister first to avoid a race with other allocating
* threads, and reregister if deallocation fails. * threads, and reregister if deallocation fails.
*/ */
extent_deregister(tsdn, extent); extent_deregister(tsdn, extent);
if (!extent_dalloc_wrapper_try(tsdn, arena, extent_hooks, if (!extent_dalloc_wrapper_try(tsdn, arena, ehooks, extent)) {
extent)) {
return; return;
} }
extent_reregister(tsdn, extent); extent_reregister(tsdn, extent);
} }
if (extent_hooks != &extent_hooks_default) { if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
} }
/* Try to decommit; purge if that fails. */ /* Try to decommit; purge if that fails. */
bool zeroed; bool zeroed;
if (!extent_committed_get(extent)) { if (!extent_committed_get(extent)) {
zeroed = true; zeroed = true;
} else if (!extent_decommit_wrapper(tsdn, arena, extent_hooks, extent, } else if (!extent_decommit_wrapper(tsdn, arena, ehooks, extent, 0,
0, extent_size_get(extent))) { extent_size_get(extent))) {
zeroed = true; zeroed = true;
} else if (extent_hooks->purge_forced != NULL && } else if (!ehooks_purge_forced(ehooks, extent_base_get(extent),
!extent_hooks->purge_forced(extent_hooks, extent_size_get(extent), 0, extent_size_get(extent),
extent_base_get(extent), extent_size_get(extent), 0, arena_ind_get(arena))) {
extent_size_get(extent), arena_ind_get(arena))) {
zeroed = true; zeroed = true;
} else if (extent_state_get(extent) == extent_state_muzzy || } else if (extent_state_get(extent) == extent_state_muzzy ||
(extent_hooks->purge_lazy != NULL && !ehooks_purge_lazy(ehooks, extent_base_get(extent),
!extent_hooks->purge_lazy(extent_hooks, extent_size_get(extent), 0, extent_size_get(extent),
extent_base_get(extent), extent_size_get(extent), 0, arena_ind_get(arena))) {
extent_size_get(extent), arena_ind_get(arena)))) {
zeroed = false; zeroed = false;
} else { } else {
zeroed = false; zeroed = false;
} }
if (extent_hooks != &extent_hooks_default) { if (!ehooks_are_default(ehooks)) {
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
extent_zeroed_set(extent, zeroed); extent_zeroed_set(extent, zeroed);
@ -1544,8 +1527,8 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_gdump_sub(tsdn, extent); extent_gdump_sub(tsdn, extent);
} }
extent_record(tsdn, arena, extent_hooks, &arena->eset_retained, extent_record(tsdn, arena, ehooks, &arena->eset_retained, extent,
extent, false); false);
} }
static void static void
@ -1562,8 +1545,8 @@ extent_destroy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
} }
void void
extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent) { extent_t *extent) {
assert(extent_base_get(extent) != NULL); assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0); assert(extent_size_get(extent) != 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
@ -1575,15 +1558,15 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
/* Try to destroy; silently fail otherwise. */ /* Try to destroy; silently fail otherwise. */
if (extent_hooks == &extent_hooks_default) { if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */ /* Call directly to propagate tsdn. */
extent_destroy_default_impl(extent_base_get(extent), extent_destroy_default_impl(extent_base_get(extent),
extent_size_get(extent)); extent_size_get(extent));
} else if (extent_hooks->destroy != NULL) { } else if (!ehooks_destroy_is_noop(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
extent_hooks->destroy(extent_hooks, ehooks_destroy(ehooks, extent_base_get(extent),
extent_base_get(extent), extent_size_get(extent), extent_size_get(extent), extent_committed_get(extent),
extent_committed_get(extent), arena_ind_get(arena)); arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
@ -1598,19 +1581,17 @@ extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
} }
static bool static bool
extent_commit_impl(tsdn_t *tsdn, arena_t *arena, extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length, bool growing_retained) {
size_t length, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0); WITNESS_RANK_CORE, growing_retained ? 1 : 0);
if (extent_hooks != &extent_hooks_default) { if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
} }
bool err = (extent_hooks->commit == NULL || bool err = ehooks_commit(ehooks, extent_base_get(extent),
extent_hooks->commit(extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena));
extent_size_get(extent), offset, length, arena_ind_get(arena))); if (!ehooks_are_default(ehooks)) {
if (extent_hooks != &extent_hooks_default) {
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
extent_committed_set(extent, extent_committed_get(extent) || !err); extent_committed_set(extent, extent_committed_get(extent) || !err);
@ -1618,11 +1599,11 @@ extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
} }
bool bool
extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset,
size_t length) { size_t length) {
return extent_commit_impl(tsdn, arena, extent_hooks, extent, offset, return extent_commit_impl(tsdn, arena, ehooks, extent, offset, length,
length, false); false);
} }
static bool static bool
@ -1633,20 +1614,17 @@ extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
} }
bool bool
extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length) {
size_t length) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
if (extent_hooks != &extent_hooks_default) { if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
} }
bool err = (extent_hooks->decommit == NULL || bool err = ehooks_decommit(ehooks, extent_base_get(extent),
extent_hooks->decommit(extent_hooks, extent_size_get(extent), offset, length, arena_ind_get(arena));
extent_base_get(extent), extent_size_get(extent), offset, length, if (!ehooks_are_default(ehooks)) {
arena_ind_get(arena)));
if (extent_hooks != &extent_hooks_default) {
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
extent_committed_set(extent, extent_committed_get(extent) && err); extent_committed_set(extent, extent_committed_get(extent) && err);
@ -1668,22 +1646,20 @@ extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
#endif #endif
static bool static bool
extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length, bool growing_retained) {
size_t length, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0); WITNESS_RANK_CORE, growing_retained ? 1 : 0);
if (extent_hooks->purge_lazy == NULL) { if (ehooks_purge_lazy_will_fail(ehooks)) {
return true; return true;
} }
if (extent_hooks != &extent_hooks_default) { if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
} }
bool err = extent_hooks->purge_lazy(extent_hooks, bool err = ehooks_purge_lazy(ehooks, extent_base_get(extent),
extent_base_get(extent), extent_size_get(extent), offset, length, extent_size_get(extent), offset, length, arena_ind_get(arena));
arena_ind_get(arena)); if (!ehooks_are_default(ehooks)) {
if (extent_hooks != &extent_hooks_default) {
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
@ -1691,11 +1667,10 @@ extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
} }
bool bool
extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length) {
size_t length) { return extent_purge_lazy_impl(tsdn, arena, ehooks, extent, offset,
return extent_purge_lazy_impl(tsdn, arena, extent_hooks, extent, length, false);
offset, length, false);
} }
#ifdef PAGES_CAN_PURGE_FORCED #ifdef PAGES_CAN_PURGE_FORCED
@ -1713,32 +1688,29 @@ extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
#endif #endif
static bool static bool
extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length, bool growing_retained) {
size_t length, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0); WITNESS_RANK_CORE, growing_retained ? 1 : 0);
if (extent_hooks->purge_forced == NULL) { if (ehooks_purge_forced_will_fail(ehooks)) {
return true; return true;
} }
if (extent_hooks != &extent_hooks_default) { if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
} }
bool err = extent_hooks->purge_forced(extent_hooks, bool err = ehooks_purge_forced(ehooks, extent_base_get(extent),
extent_base_get(extent), extent_size_get(extent), offset, length, extent_size_get(extent), offset, length, arena_ind_get(arena));
arena_ind_get(arena)); if (!ehooks_are_default(ehooks)) {
if (extent_hooks != &extent_hooks_default) {
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
return err; return err;
} }
bool bool
extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_t *extent, size_t offset, size_t length) {
size_t length) { return extent_purge_forced_impl(tsdn, arena, ehooks, extent,
return extent_purge_forced_impl(tsdn, arena, extent_hooks, extent,
offset, length, false); offset, length, false);
} }
@ -1765,15 +1737,14 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
* and returns the trail (except in case of error). * and returns the trail (except in case of error).
*/ */
static extent_t * static extent_t *
extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b, size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained) {
bool growing_retained) {
assert(extent_size_get(extent) == size_a + size_b); assert(extent_size_get(extent) == size_a + size_b);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0); WITNESS_RANK_CORE, growing_retained ? 1 : 0);
if (extent_hooks->split == NULL) { if (ehooks_split_will_fail(ehooks)) {
return NULL; return NULL;
} }
@ -1815,13 +1786,13 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena,
extent_lock2(tsdn, extent, trail); extent_lock2(tsdn, extent, trail);
if (extent_hooks != &extent_hooks_default) { if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
} }
bool err = extent_hooks->split(extent_hooks, extent_base_get(extent), bool err = ehooks_split(ehooks, extent_base_get(extent),
size_a + size_b, size_a, size_b, extent_committed_get(extent), size_a + size_b, size_a, size_b, extent_committed_get(extent),
arena_ind_get(arena)); arena_ind_get(arena));
if (extent_hooks != &extent_hooks_default) { if (!ehooks_are_default(ehooks)) {
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
if (err) { if (err) {
@ -1848,11 +1819,11 @@ label_error_a:
} }
extent_t * extent_t *
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) { size_t size_b, szind_t szind_b, bool slab_b) {
return extent_split_impl(tsdn, arena, extent_hooks, extent, size_a, return extent_split_impl(tsdn, arena, ehooks, extent, size_a, szind_a,
szind_a, slab_a, size_b, szind_b, slab_b, false); slab_a, size_b, szind_b, slab_b, false);
} }
static bool static bool
@ -1914,28 +1885,26 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
} }
static bool static bool
extent_merge_impl(tsdn_t *tsdn, arena_t *arena, extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b, extent_t *b, bool growing_retained) {
bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0); WITNESS_RANK_CORE, growing_retained ? 1 : 0);
assert(extent_base_get(a) < extent_base_get(b)); assert(extent_base_get(a) < extent_base_get(b));
if (extent_hooks->merge == NULL || extent_head_no_merge(a, b)) { if (ehooks_merge_will_fail(ehooks) || extent_head_no_merge(a, b)) {
return true; return true;
} }
bool err; bool err;
if (extent_hooks == &extent_hooks_default) { if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */ /* Call directly to propagate tsdn. */
err = extent_merge_default_impl(extent_base_get(a), err = extent_merge_default_impl(extent_base_get(a),
extent_base_get(b)); extent_base_get(b));
} else { } else {
extent_hook_pre_reentrancy(tsdn, arena); extent_hook_pre_reentrancy(tsdn, arena);
err = extent_hooks->merge(extent_hooks, err = ehooks_merge(ehooks, extent_base_get(a),
extent_base_get(a), extent_size_get(a), extent_base_get(b), extent_size_get(a), extent_base_get(b), extent_size_get(b),
extent_size_get(b), extent_committed_get(a), extent_committed_get(a), arena_ind_get(arena));
arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn); extent_hook_post_reentrancy(tsdn);
} }
@ -1991,9 +1960,9 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
} }
bool bool
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b) { extent_t *a, extent_t *b) {
return extent_merge_impl(tsdn, arena, extent_hooks, a, b, false); return extent_merge_impl(tsdn, arena, ehooks, a, b, false);
} }
bool bool

7
src/extent_dss.c
View File

@ -194,9 +194,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
*commit = pages_decommit(ret, size); *commit = pages_decommit(ret, size);
} }
if (*zero && *commit) { if (*zero && *commit) {
extent_hooks_t *extent_hooks =
arena_get_extent_hooks(arena);
extent_t extent; extent_t extent;
ehooks_t *ehooks = arena_get_ehooks(
arena);
extent_init(&extent, extent_init(&extent,
arena_ind_get(arena), ret, size, arena_ind_get(arena), ret, size,
@ -204,8 +204,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
extent_state_active, false, true, extent_state_active, false, true,
true, EXTENT_NOT_HEAD); true, EXTENT_NOT_HEAD);
if (extent_purge_forced_wrapper(tsdn, if (extent_purge_forced_wrapper(tsdn,
arena, extent_hooks, &extent, 0, arena, ehooks, &extent, 0, size)) {
size)) {
memset(ret, 0, size); memset(ret, 0, size);
} }
} }

36
src/large.c
View File

@ -93,20 +93,20 @@ static bool
large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
arena_t *arena = arena_get_from_extent(extent); arena_t *arena = arena_get_from_extent(extent);
size_t oldusize = extent_usize_get(extent); size_t oldusize = extent_usize_get(extent);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
size_t diff = extent_size_get(extent) - (usize + sz_large_pad); size_t diff = extent_size_get(extent) - (usize + sz_large_pad);
assert(oldusize > usize); assert(oldusize > usize);
if (extent_hooks->split == NULL) { if (ehooks_split_will_fail(ehooks)) {
return true; return true;
} }
/* Split excess pages. */ /* Split excess pages. */
if (diff != 0) { if (diff != 0) {
extent_t *trail = extent_split_wrapper(tsdn, arena, extent_t *trail = extent_split_wrapper(tsdn, arena,
extent_hooks, extent, usize + sz_large_pad, ehooks, extent, usize + sz_large_pad, sz_size2index(usize),
sz_size2index(usize), false, diff, SC_NSIZES, false); false, diff, SC_NSIZES, false);
if (trail == NULL) { if (trail == NULL) {
return true; return true;
} }
@ -116,7 +116,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
extent_size_get(trail)); extent_size_get(trail));
} }
arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, trail); arena_extents_dirty_dalloc(tsdn, arena, ehooks, trail);
} }
arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize); arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize);
@ -129,10 +129,10 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool zero) { bool zero) {
arena_t *arena = arena_get_from_extent(extent); arena_t *arena = arena_get_from_extent(extent);
size_t oldusize = extent_usize_get(extent); size_t oldusize = extent_usize_get(extent);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
size_t trailsize = usize - oldusize; size_t trailsize = usize - oldusize;
if (extent_hooks->merge == NULL) { if (ehooks_merge_will_fail(ehooks)) {
return true; return true;
} }
@ -149,17 +149,17 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool commit = true; bool commit = true;
extent_t *trail; extent_t *trail;
bool new_mapping; bool new_mapping;
if ((trail = extents_alloc(tsdn, arena, extent_hooks, if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
&arena->eset_dirty, extent_past_get(extent), trailsize, 0, extent_past_get(extent), trailsize, 0, CACHELINE, false, SC_NSIZES,
CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL &is_zeroed_trail, &commit)) != NULL
|| (trail = extents_alloc(tsdn, arena, extent_hooks, || (trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
&arena->eset_muzzy, extent_past_get(extent), trailsize, 0, extent_past_get(extent), trailsize, 0, CACHELINE, false, SC_NSIZES,
CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) { &is_zeroed_trail, &commit)) != NULL) {
if (config_stats) { if (config_stats) {
new_mapping = false; new_mapping = false;
} }
} else { } else {
if ((trail = extent_alloc_wrapper(tsdn, arena, extent_hooks, if ((trail = extent_alloc_wrapper(tsdn, arena, ehooks,
extent_past_get(extent), trailsize, 0, CACHELINE, false, extent_past_get(extent), trailsize, 0, CACHELINE, false,
SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) { SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) {
return true; return true;
@ -169,8 +169,8 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
} }
} }
if (extent_merge_wrapper(tsdn, arena, extent_hooks, extent, trail)) { if (extent_merge_wrapper(tsdn, arena, ehooks, extent, trail)) {
extent_dalloc_wrapper(tsdn, arena, extent_hooks, trail); extent_dalloc_wrapper(tsdn, arena, ehooks, trail);
return true; return true;
} }
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
@ -339,8 +339,8 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
static void static void
large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena); ehooks_t *ehooks = arena_get_ehooks(arena);
arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, extent); arena_extents_dirty_dalloc(tsdn, arena, ehooks, extent);
} }
void void