diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 74daf6a8..bfc0073b 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1551,7 +1551,7 @@ malloc_conf = "xmalloc:true";]]> arena.<i>.extent_hooks - (extent_hooks_t) + (extent_hooks_t *) rw Get or set the extent management hook functions for @@ -1567,7 +1567,8 @@ malloc_conf = "xmalloc:true";]]> allocation. +};]]> The extent_hooks_t structure comprises function pointers which are described individually below. jemalloc uses these functions to manage extent lifetime, which starts off with allocation of @@ -1592,6 +1593,7 @@ typedef struct { typedef void *(extent_alloc_t) + extent_hooks_t *extent_hooks void *new_addr size_t size size_t alignment @@ -1627,6 +1629,7 @@ typedef struct { typedef bool (extent_dalloc_t) + extent_hooks_t *extent_hooks void *addr size_t size bool committed @@ -1646,6 +1649,7 @@ typedef struct { typedef bool (extent_commit_t) + extent_hooks_t *extent_hooks void *addr size_t size size_t offset @@ -1667,6 +1671,7 @@ typedef struct { typedef bool (extent_decommit_t) + extent_hooks_t *extent_hooks void *addr size_t size size_t offset @@ -1688,6 +1693,7 @@ typedef struct { typedef bool (extent_purge_t) + extent_hooks_t *extent_hooks void *addr size_tsize size_t offset @@ -1707,6 +1713,7 @@ typedef struct { typedef bool (extent_split_t) + extent_hooks_t *extent_hooks void *addr size_t size size_t size_a @@ -1728,6 +1735,7 @@ typedef struct { typedef bool (extent_merge_t) + extent_hooks_t *extent_hooks void *addr_a size_t size_a void *addr_b diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index fc0a755f..3c931c34 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -240,11 +240,15 @@ struct arena_s { */ extent_heap_t extents_cached[NPSIZES]; extent_heap_t extents_retained[NPSIZES]; - /* User-configurable extent hook functions. */ - extent_hooks_t extent_hooks; - /* Protects extents_cached, extents_retained, and extent_hooks. */ + /* Protects extents_cached and extents_retained. */ malloc_mutex_t extents_mtx; + /* User-configurable extent hook functions. */ + union { + extent_hooks_t *extent_hooks; + void *extent_hooks_pun; + }; + /* Cache of extent structures that were allocated via base_alloc(). */ ql_head(extent_t) extent_cache; malloc_mutex_t extent_cache_mtx; @@ -279,10 +283,10 @@ extern ssize_t opt_decay_time; extern const arena_bin_info_t arena_bin_info[NBINS]; extent_t *arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, - bool *zero); + extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, + size_t alignment, bool *zero); void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent); + extent_hooks_t **r_extent_hooks, extent_t *extent); void arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache); void arena_extent_cache_maybe_remove(arena_t *arena, extent_t *extent, diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 8b8dbe80..6e155206 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -3,15 +3,7 @@ typedef struct extent_s extent_t; -#define EXTENT_HOOKS_INITIALIZER { \ - NULL, \ - NULL, \ - NULL, \ - NULL, \ - NULL, \ - NULL, \ - NULL \ -} +#define EXTENT_HOOKS_INITIALIZER NULL #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ @@ -93,9 +85,8 @@ extern const extent_hooks_t extent_hooks_default; extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent); -extent_hooks_t extent_hooks_get(tsdn_t *tsdn, arena_t *arena); -extent_hooks_t extent_hooks_set(tsdn_t *tsdn, arena_t *arena, - const extent_hooks_t *extent_hooks); +extent_hooks_t *extent_hooks_get(arena_t *arena); +extent_hooks_t *extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks); #ifdef JEMALLOC_JET typedef size_t (extent_size_quantize_t)(size_t); @@ -109,29 +100,29 @@ size_t extent_size_quantize_ceil(size_t size); ph_proto(, extent_heap_, extent_heap_t, extent_t) extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, + extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool slab); extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, + extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab); void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent); + extent_hooks_t **r_extent_hooks, extent_t *extent); void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent); + extent_hooks_t **r_extent_hooks, extent_t *extent); bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length); bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length); bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length); extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, size_t usize_a, size_t size_b, size_t usize_b); bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *a, extent_t *b); + extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b); void extent_prefork(tsdn_t *tsdn); void extent_postfork_parent(tsdn_t *tsdn); void extent_postfork_child(tsdn_t *tsdn); diff --git a/include/jemalloc/jemalloc_typedefs.h.in b/include/jemalloc/jemalloc_typedefs.h.in index 99f07ab2..e5ba7166 100644 --- a/include/jemalloc/jemalloc_typedefs.h.in +++ b/include/jemalloc/jemalloc_typedefs.h.in @@ -1,53 +1,61 @@ +typedef struct extent_hooks_s extent_hooks_t; + /* * void * * extent_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, * bool *commit, unsigned arena_ind); */ -typedef void *(extent_alloc_t)(void *, size_t, size_t, bool *, bool *, - unsigned); +typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *, + bool *, unsigned); /* * bool * extent_dalloc(void *addr, size_t size, bool committed, unsigned arena_ind); */ -typedef bool (extent_dalloc_t)(void *, size_t, bool, unsigned); +typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); /* * bool * extent_commit(void *addr, size_t size, size_t offset, size_t length, * unsigned arena_ind); */ -typedef bool (extent_commit_t)(void *, size_t, size_t, size_t, unsigned); +typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); /* * bool * extent_decommit(void *addr, size_t size, size_t offset, size_t length, * unsigned arena_ind); */ -typedef bool (extent_decommit_t)(void *, size_t, size_t, size_t, unsigned); +typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t, + size_t, unsigned); /* * bool * extent_purge(void *addr, size_t size, size_t offset, size_t length, * unsigned arena_ind); */ -typedef bool (extent_purge_t)(void *, size_t, size_t, size_t, unsigned); +typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); /* * bool * extent_split(void *addr, size_t size, size_t size_a, size_t size_b, * bool committed, unsigned arena_ind); */ -typedef bool (extent_split_t)(void *, size_t, size_t, size_t, bool, unsigned); +typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + bool, unsigned); /* * bool * extent_merge(void *addr_a, size_t size_a, void *addr_b, size_t size_b, * bool committed, unsigned arena_ind); */ -typedef bool (extent_merge_t)(void *, size_t, void *, size_t, bool, unsigned); +typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, + bool, unsigned); -typedef struct { +struct extent_hooks_s { extent_alloc_t *alloc; extent_dalloc_t *dalloc; extent_commit_t *commit; @@ -55,4 +63,4 @@ typedef struct { extent_purge_t *purge; extent_split_t *split; extent_merge_t *merge; -} extent_hooks_t; +}; diff --git a/src/arena.c b/src/arena.c index c77db5d0..32e1915c 100644 --- a/src/arena.c +++ b/src/arena.c @@ -54,25 +54,25 @@ arena_extent_dirty_npages(const extent_t *extent) static extent_t * arena_extent_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, + extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool slab) { malloc_mutex_assert_owner(tsdn, &arena->lock); - return (extent_alloc_cache(tsdn, arena, extent_hooks, new_addr, usize, + return (extent_alloc_cache(tsdn, arena, r_extent_hooks, new_addr, usize, pad, alignment, zero, slab)); } extent_t * arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, - bool *zero) + extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, + size_t alignment, bool *zero) { extent_t *extent; malloc_mutex_lock(tsdn, &arena->lock); - extent = arena_extent_cache_alloc_locked(tsdn, arena, extent_hooks, + extent = arena_extent_cache_alloc_locked(tsdn, arena, r_extent_hooks, new_addr, size, 0, alignment, zero, false); malloc_mutex_unlock(tsdn, &arena->lock); @@ -81,22 +81,22 @@ arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena, static void arena_extent_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent) + extent_hooks_t **r_extent_hooks, extent_t *extent) { malloc_mutex_assert_owner(tsdn, &arena->lock); - extent_dalloc_cache(tsdn, arena, extent_hooks, extent); + extent_dalloc_cache(tsdn, arena, r_extent_hooks, extent); arena_maybe_purge(tsdn, arena); } void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent) + extent_hooks_t **r_extent_hooks, extent_t *extent) { malloc_mutex_lock(tsdn, &arena->lock); - arena_extent_cache_dalloc_locked(tsdn, arena, extent_hooks, extent); + arena_extent_cache_dalloc_locked(tsdn, arena, r_extent_hooks, extent); malloc_mutex_unlock(tsdn, &arena->lock); } @@ -321,12 +321,12 @@ arena_large_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize) static extent_t * arena_extent_alloc_large_hard(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, size_t usize, size_t alignment, bool *zero) + extent_hooks_t **r_extent_hooks, size_t usize, size_t alignment, bool *zero) { extent_t *extent; bool commit = true; - extent = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, usize, + extent = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, usize, large_pad, alignment, zero, &commit, false); if (extent == NULL) { /* Revert optimistic stats updates. */ @@ -347,7 +347,7 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero) { extent_t *extent; - extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER; + extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; malloc_mutex_lock(tsdn, &arena->lock); @@ -373,7 +373,7 @@ void arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent, bool locked) { - extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER; + extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; if (!locked) malloc_mutex_lock(tsdn, &arena->lock); @@ -735,7 +735,7 @@ arena_dirty_count(tsdn_t *tsdn, arena_t *arena) } static size_t -arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, +arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, size_t ndirty_limit, extent_t *purge_extents_sentinel) { extent_t *extent, *next; @@ -757,7 +757,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, /* Allocate. */ zero = false; textent = arena_extent_cache_alloc_locked(tsdn, arena, - extent_hooks, extent_base_get(extent), + r_extent_hooks, extent_base_get(extent), extent_size_get(extent), 0, CACHELINE, &zero, false); assert(textent == extent); assert(zero == extent_zeroed_get(extent)); @@ -774,8 +774,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, } static size_t -arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - extent_t *purge_extents_sentinel) +arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extent_t *purge_extents_sentinel) { UNUSED size_t nmadvise; size_t npurged; @@ -793,7 +793,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, next = qr_next(extent, qr_link); extent_ring_remove(extent); - extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent); + extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, extent); } if (config_stats) { @@ -816,7 +816,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) { - extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena); + extent_hooks_t *extent_hooks = extent_hooks_get(arena); size_t npurge, npurged; extent_t purge_extents_sentinel; @@ -866,7 +866,7 @@ arena_purge(tsdn_t *tsdn, arena_t *arena, bool all) static void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) { - extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER; + extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab); @@ -988,7 +988,7 @@ arena_bin_slabs_full_remove(extent_t *slab) static extent_t * arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, const arena_bin_info_t *bin_info) + extent_hooks_t **r_extent_hooks, const arena_bin_info_t *bin_info) { extent_t *slab; bool zero, commit; @@ -996,7 +996,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, zero = false; commit = true; malloc_mutex_unlock(tsdn, &arena->lock); - slab = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, + slab = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, bin_info->slab_size, 0, PAGE, &zero, &commit, true); malloc_mutex_lock(tsdn, &arena->lock); @@ -1009,7 +1009,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, { extent_t *slab; arena_slab_data_t *slab_data; - extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER; + extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; bool zero; zero = false; @@ -1796,7 +1796,7 @@ arena_new(tsdn_t *tsdn, unsigned ind) extent_heap_new(&arena->extents_retained[i]); } - arena->extent_hooks = extent_hooks_default; + arena->extent_hooks = (extent_hooks_t *)&extent_hooks_default; if (malloc_mutex_init(&arena->extents_mtx, "arena_extents", WITNESS_RANK_ARENA_EXTENTS)) diff --git a/src/ctl.c b/src/ctl.c index b91ea135..813d5fab 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1646,15 +1646,15 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, if (arena_ind < narenas_total_get() && (arena = arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { if (newp != NULL) { - extent_hooks_t old_extent_hooks, new_extent_hooks; - WRITE(new_extent_hooks, extent_hooks_t); - old_extent_hooks = extent_hooks_set(tsd_tsdn(tsd), - arena, &new_extent_hooks); - READ(old_extent_hooks, extent_hooks_t); + extent_hooks_t *old_extent_hooks, *new_extent_hooks; + WRITE(new_extent_hooks, extent_hooks_t *); + old_extent_hooks = extent_hooks_set(arena, + new_extent_hooks); + READ(old_extent_hooks, extent_hooks_t *); } else { - extent_hooks_t old_extent_hooks = - extent_hooks_get(tsd_tsdn(tsd), arena); - READ(old_extent_hooks, extent_hooks_t); + extent_hooks_t *old_extent_hooks = + extent_hooks_get(arena); + READ(old_extent_hooks, extent_hooks_t *); } } else { ret = EFAULT; diff --git a/src/extent.c b/src/extent.c index 0c41c066..0c66e31a 100644 --- a/src/extent.c +++ b/src/extent.c @@ -6,20 +6,23 @@ rtree_t extents_rtree; -static void *extent_alloc_default(void *new_addr, size_t size, - size_t alignment, bool *zero, bool *commit, unsigned arena_ind); -static bool extent_dalloc_default(void *addr, size_t size, bool committed, +static void *extent_alloc_default(extent_hooks_t *extent_hooks, + void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, + unsigned arena_ind); +static bool extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, + size_t size, bool committed, unsigned arena_ind); +static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr, + size_t size, size_t offset, size_t length, unsigned arena_ind); +static bool extent_decommit_default(extent_hooks_t *extent_hooks, + void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); +static bool extent_purge_default(extent_hooks_t *extent_hooks, void *addr, + size_t size, size_t offset, size_t length, unsigned arena_ind); +static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, + size_t size, size_t size_a, size_t size_b, bool committed, + unsigned arena_ind); +static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, + size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind); -static bool extent_commit_default(void *addr, size_t size, size_t offset, - size_t length, unsigned arena_ind); -static bool extent_decommit_default(void *addr, size_t size, size_t offset, - size_t length, unsigned arena_ind); -static bool extent_purge_default(void *addr, size_t size, size_t offset, - size_t length, unsigned arena_ind); -static bool extent_split_default(void *addr, size_t size, size_t size_a, - size_t size_b, bool committed, unsigned arena_ind); -static bool extent_merge_default(void *addr_a, size_t size_a, void *addr_b, - size_t size_b, bool committed, unsigned arena_ind); const extent_hooks_t extent_hooks_default = { extent_alloc_default, @@ -42,7 +45,7 @@ static size_t highchunks; */ static void extent_record(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_heap_t extent_heaps[NPSIZES], + extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent); /******************************************************************************/ @@ -73,89 +76,34 @@ extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); } -static extent_hooks_t -extent_hooks_get_locked(arena_t *arena) +extent_hooks_t * +extent_hooks_get(arena_t *arena) { - return (arena->extent_hooks); + return ((extent_hooks_t *)atomic_read_p(&arena->extent_hooks_pun)); } -extent_hooks_t -extent_hooks_get(tsdn_t *tsdn, arena_t *arena) +extent_hooks_t * +extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks) { - extent_hooks_t extent_hooks; + extent_hooks_t *old_extent_hooks = extent_hooks_get(arena); + union { + extent_hooks_t **h; + void **v; + } u; - malloc_mutex_lock(tsdn, &arena->extents_mtx); - extent_hooks = extent_hooks_get_locked(arena); - malloc_mutex_unlock(tsdn, &arena->extents_mtx); - - return (extent_hooks); -} - -extent_hooks_t -extent_hooks_set(tsdn_t *tsdn, arena_t *arena, - const extent_hooks_t *extent_hooks) -{ - extent_hooks_t old_extent_hooks; - - malloc_mutex_lock(tsdn, &arena->extents_mtx); - old_extent_hooks = arena->extent_hooks; - /* - * Copy each field atomically so that it is impossible for readers to - * see partially updated pointers. There are places where readers only - * need one hook function pointer (therefore no need to copy the - * entirety of arena->extent_hooks), and stale reads do not affect - * correctness, so they perform unlocked reads. - */ -#define ATOMIC_COPY_HOOK(n) do { \ - union { \ - extent_##n##_t **n; \ - void **v; \ - } u; \ - u.n = &arena->extent_hooks.n; \ - atomic_write_p(u.v, extent_hooks->n); \ -} while (0) - ATOMIC_COPY_HOOK(alloc); - ATOMIC_COPY_HOOK(dalloc); - ATOMIC_COPY_HOOK(commit); - ATOMIC_COPY_HOOK(decommit); - ATOMIC_COPY_HOOK(purge); - ATOMIC_COPY_HOOK(split); - ATOMIC_COPY_HOOK(merge); -#undef ATOMIC_COPY_HOOK - malloc_mutex_unlock(tsdn, &arena->extents_mtx); + u.h = &arena->extent_hooks; + atomic_write_p(u.v, extent_hooks); return (old_extent_hooks); } static void -extent_hooks_assure_initialized_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, bool locked) -{ - static const extent_hooks_t uninitialized_hooks = - EXTENT_HOOKS_INITIALIZER; - - if (memcmp(extent_hooks, &uninitialized_hooks, sizeof(extent_hooks_t)) - == 0) { - *extent_hooks = locked ? extent_hooks_get_locked(arena) : - extent_hooks_get(tsdn, arena); - } -} - -static void -extent_hooks_assure_initialized_locked(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks) +extent_hooks_assure_initialized(arena_t *arena, extent_hooks_t **r_extent_hooks) { - extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, true); -} - -static void -extent_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks) -{ - - extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, false); + if (*r_extent_hooks == EXTENT_HOOKS_INITIALIZER) + *r_extent_hooks = extent_hooks_get(arena); } #ifdef JEMALLOC_JET @@ -409,7 +357,7 @@ extent_first_best_fit(arena_t *arena, extent_heap_t extent_heaps[NPSIZES], } static void -extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, +extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, bool cache, extent_t *extent) { @@ -418,14 +366,14 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, * that this is only a virtual memory leak. */ if (cache) { - extent_purge_wrapper(tsdn, arena, extent_hooks, extent, 0, + extent_purge_wrapper(tsdn, arena, r_extent_hooks, extent, 0, extent_size_get(extent)); } extent_dalloc(tsdn, arena, extent); } static extent_t * -extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, +extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) @@ -444,7 +392,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, if (alloc_size < usize) return (NULL); malloc_mutex_lock(tsdn, &arena->extents_mtx); - extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks); + extent_hooks_assure_initialized(arena, r_extent_hooks); if (new_addr != NULL) { rtree_elm_t *elm; @@ -482,10 +430,11 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, /* Split the lead. */ if (leadsize != 0) { extent_t *lead = extent; - extent = extent_split_wrapper(tsdn, arena, extent_hooks, lead, - leadsize, leadsize, size + trailsize, usize + trailsize); + extent = extent_split_wrapper(tsdn, arena, r_extent_hooks, + lead, leadsize, leadsize, size + trailsize, usize + + trailsize); if (extent == NULL) { - extent_leak(tsdn, arena, extent_hooks, cache, lead); + extent_leak(tsdn, arena, r_extent_hooks, cache, lead); malloc_mutex_unlock(tsdn, &arena->extents_mtx); return (NULL); } @@ -496,9 +445,10 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, /* Split the trail. */ if (trailsize != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, - extent_hooks, extent, size, usize, trailsize, trailsize); + r_extent_hooks, extent, size, usize, trailsize, trailsize); if (trail == NULL) { - extent_leak(tsdn, arena, extent_hooks, cache, extent); + extent_leak(tsdn, arena, r_extent_hooks, cache, + extent); malloc_mutex_unlock(tsdn, &arena->extents_mtx); return (NULL); } @@ -513,10 +463,10 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, } if (!extent_committed_get(extent) && - extent_hooks->commit(extent_base_get(extent), + (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) { malloc_mutex_unlock(tsdn, &arena->extents_mtx); - extent_record(tsdn, arena, extent_hooks, extent_heaps, cache, + extent_record(tsdn, arena, r_extent_hooks, extent_heaps, cache, extent); return (NULL); } @@ -582,9 +532,9 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, } extent_t * -extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, - bool slab) +extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, + size_t alignment, bool *zero, bool slab) { extent_t *extent; bool commit; @@ -593,7 +543,7 @@ extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, assert(alignment != 0); commit = true; - extent = extent_recycle(tsdn, arena, extent_hooks, + extent = extent_recycle(tsdn, arena, r_extent_hooks, arena->extents_cached, true, new_addr, usize, pad, alignment, zero, &commit, slab); if (extent == NULL) @@ -603,13 +553,15 @@ extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, } static void * -extent_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, - bool *commit, unsigned arena_ind) +extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, + size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { void *ret; tsdn_t *tsdn; arena_t *arena; + assert(extent_hooks == &extent_hooks_default); + tsdn = tsdn_fetch(); arena = arena_get(tsdn, arena_ind, false); /* @@ -627,7 +579,7 @@ extent_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, static extent_t * extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, + extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) { extent_t *extent; @@ -635,7 +587,7 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, assert(usize != 0); assert(alignment != 0); - extent = extent_recycle(tsdn, arena, extent_hooks, + extent = extent_recycle(tsdn, arena, r_extent_hooks, arena->extents_retained, false, new_addr, usize, pad, alignment, zero, commit, slab); if (extent != NULL && config_stats) { @@ -648,7 +600,7 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, static extent_t * extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, + extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) { extent_t *extent; @@ -659,8 +611,8 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, extent = extent_alloc(tsdn, arena); if (extent == NULL) return (NULL); - addr = extent_hooks->alloc(new_addr, size, alignment, zero, commit, - arena->ind); + addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, size, + alignment, zero, commit, arena->ind); if (addr == NULL) { extent_dalloc(tsdn, arena, extent); return (NULL); @@ -669,7 +621,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, if (pad != 0) extent_addr_randomize(tsdn, extent, alignment); if (extent_register(tsdn, extent)) { - extent_leak(tsdn, arena, extent_hooks, false, extent); + extent_leak(tsdn, arena, r_extent_hooks, false, extent); return (NULL); } @@ -677,18 +629,18 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, } extent_t * -extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, - bool *commit, bool slab) +extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, + size_t alignment, bool *zero, bool *commit, bool slab) { extent_t *extent; - extent_hooks_assure_initialized(tsdn, arena, extent_hooks); + extent_hooks_assure_initialized(arena, r_extent_hooks); - extent = extent_alloc_retained(tsdn, arena, extent_hooks, new_addr, + extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, new_addr, usize, pad, alignment, zero, commit, slab); if (extent == NULL) { - extent = extent_alloc_wrapper_hard(tsdn, arena, extent_hooks, + extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks, new_addr, usize, pad, alignment, zero, commit, slab); } @@ -712,8 +664,9 @@ extent_can_coalesce(const extent_t *a, const extent_t *b) } static void -extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - extent_t *a, extent_t *b, extent_heap_t extent_heaps[NPSIZES], bool cache) +extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, + extent_heap_t extent_heaps[NPSIZES], bool cache) { if (!extent_can_coalesce(a, b)) @@ -725,7 +678,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, arena_extent_cache_maybe_remove(extent_arena_get(a), a, cache); arena_extent_cache_maybe_remove(extent_arena_get(b), b, cache); - if (extent_merge_wrapper(tsdn, arena, extent_hooks, a, b)) { + if (extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b)) { extent_heaps_insert(extent_heaps, a); extent_heaps_insert(extent_heaps, b); arena_extent_cache_maybe_insert(extent_arena_get(a), a, cache); @@ -738,7 +691,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, } static void -extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, +extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent) { extent_t *prev, *next; @@ -748,7 +701,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, assert(!cache || !extent_zeroed_get(extent)); malloc_mutex_lock(tsdn, &arena->extents_mtx); - extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks); + extent_hooks_assure_initialized(arena, r_extent_hooks); extent_usize_set(extent, 0); extent_active_set(extent, false); @@ -766,7 +719,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, next = rtree_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_past_get(extent), false); if (next != NULL) { - extent_try_coalesce(tsdn, arena, extent_hooks, extent, next, + extent_try_coalesce(tsdn, arena, r_extent_hooks, extent, next, extent_heaps, cache); } @@ -774,7 +727,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, prev = rtree_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_before_get(extent), false); if (prev != NULL) { - extent_try_coalesce(tsdn, arena, extent_hooks, prev, extent, + extent_try_coalesce(tsdn, arena, r_extent_hooks, prev, extent, extent_heaps, cache); } @@ -782,8 +735,8 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, } void -extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - extent_t *extent) +extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extent_t *extent) { assert(extent_base_get(extent) != NULL); @@ -792,15 +745,17 @@ extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_addr_set(extent, extent_base_get(extent)); extent_zeroed_set(extent, false); - extent_record(tsdn, arena, extent_hooks, arena->extents_cached, true, + extent_record(tsdn, arena, r_extent_hooks, arena->extents_cached, true, extent); } static bool -extent_dalloc_default(void *addr, size_t size, bool committed, - unsigned arena_ind) +extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size, + bool committed, unsigned arena_ind) { + assert(extent_hooks == &extent_hooks_default); + if (!have_dss || !extent_in_dss(tsdn_fetch(), addr)) return (extent_dalloc_mmap(addr, size)); return (true); @@ -808,7 +763,7 @@ extent_dalloc_default(void *addr, size_t size, bool committed, void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent) + extent_hooks_t **r_extent_hooks, extent_t *extent) { assert(extent_base_get(extent) != NULL); @@ -816,9 +771,9 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_addr_set(extent, extent_base_get(extent)); - extent_hooks_assure_initialized(tsdn, arena, extent_hooks); + extent_hooks_assure_initialized(arena, r_extent_hooks); /* Try to deallocate. */ - if (!extent_hooks->dalloc(extent_base_get(extent), + if (!(*r_extent_hooks)->dalloc(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), extent_committed_get(extent), arena->ind)) { extent_deregister(tsdn, extent); @@ -828,66 +783,73 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, /* Try to decommit; purge if that fails. */ if (extent_committed_get(extent)) { extent_committed_set(extent, - extent_hooks->decommit(extent_base_get(extent), - extent_size_get(extent), 0, extent_size_get(extent), - arena->ind)); + (*r_extent_hooks)->decommit(*r_extent_hooks, + extent_base_get(extent), extent_size_get(extent), 0, + extent_size_get(extent), arena->ind)); } extent_zeroed_set(extent, !extent_committed_get(extent) || - !extent_hooks->purge(extent_base_get(extent), + !(*r_extent_hooks)->purge(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), 0, extent_size_get(extent), arena->ind)); if (config_stats) arena->stats.retained += extent_size_get(extent); - extent_record(tsdn, arena, extent_hooks, arena->extents_retained, false, - extent); + extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained, + false, extent); } static bool -extent_commit_default(void *addr, size_t size, size_t offset, size_t length, - unsigned arena_ind) +extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { + assert(extent_hooks == &extent_hooks_default); + return (pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset), length)); } bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { - extent_hooks_assure_initialized(tsdn, arena, extent_hooks); - return (extent_hooks->commit(extent_base_get(extent), - extent_size_get(extent), offset, length, arena->ind)); + extent_hooks_assure_initialized(arena, r_extent_hooks); + return ((*r_extent_hooks)->commit(*r_extent_hooks, + extent_base_get(extent), extent_size_get(extent), offset, length, + arena->ind)); } static bool -extent_decommit_default(void *addr, size_t size, size_t offset, size_t length, - unsigned arena_ind) +extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { + assert(extent_hooks == &extent_hooks_default); + return (pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset), length)); } bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { - extent_hooks_assure_initialized(tsdn, arena, extent_hooks); - return (extent_hooks->decommit(extent_base_get(extent), - extent_size_get(extent), offset, length, arena->ind)); + extent_hooks_assure_initialized(arena, r_extent_hooks); + return ((*r_extent_hooks)->decommit(*r_extent_hooks, + extent_base_get(extent), extent_size_get(extent), offset, length, + arena->ind)); } static bool -extent_purge_default(void *addr, size_t size, size_t offset, size_t length, - unsigned arena_ind) +extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { + assert(extent_hooks == &extent_hooks_default); assert(addr != NULL); assert((offset & PAGE_MASK) == 0); assert(length != 0); @@ -898,29 +860,33 @@ extent_purge_default(void *addr, size_t size, size_t offset, size_t length, } bool -extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - extent_t *extent, size_t offset, size_t length) +extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, + size_t length) { - extent_hooks_assure_initialized(tsdn, arena, extent_hooks); - return (extent_hooks->purge(extent_base_get(extent), - extent_size_get(extent), offset, length, arena->ind)); + extent_hooks_assure_initialized(arena, r_extent_hooks); + return ((*r_extent_hooks)->purge(*r_extent_hooks, + extent_base_get(extent), extent_size_get(extent), offset, length, + arena->ind)); } static bool -extent_split_default(void *addr, size_t size, size_t size_a, size_t size_b, - bool committed, unsigned arena_ind) +extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { + assert(extent_hooks == &extent_hooks_default); + if (!maps_coalesce) return (true); return (false); } extent_t * -extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - extent_t *extent, size_t size_a, size_t usize_a, size_t size_b, - size_t usize_b) +extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, + size_t usize_a, size_t size_b, size_t usize_b) { extent_t *trail; rtree_ctx_t rtree_ctx_fallback; @@ -929,7 +895,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, assert(extent_size_get(extent) == size_a + size_b); - extent_hooks_assure_initialized(tsdn, arena, extent_hooks); + extent_hooks_assure_initialized(arena, r_extent_hooks); trail = extent_alloc(tsdn, arena); if (trail == NULL) @@ -956,8 +922,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, &trail_elm_a, &trail_elm_b)) goto label_error_c; - if (extent_hooks->split(extent_base_get(extent), size_a + size_b, - size_a, size_b, extent_committed_get(extent), arena->ind)) + if ((*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent), + size_a + size_b, size_a, size_b, extent_committed_get(extent), + arena->ind)) goto label_error_d; extent_size_set(extent, size_a); @@ -981,10 +948,12 @@ label_error_a: } static bool -extent_merge_default(void *addr_a, size_t size_a, void *addr_b, size_t size_b, - bool committed, unsigned arena_ind) +extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { + assert(extent_hooks == &extent_hooks_default); + if (!maps_coalesce) return (true); if (have_dss) { @@ -997,17 +966,17 @@ extent_merge_default(void *addr_a, size_t size_a, void *addr_b, size_t size_b, } bool -extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, - extent_t *a, extent_t *b) +extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) { rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; - extent_hooks_assure_initialized(tsdn, arena, extent_hooks); - if (extent_hooks->merge(extent_base_get(a), extent_size_get(a), - extent_base_get(b), extent_size_get(b), extent_committed_get(a), - arena->ind)) + extent_hooks_assure_initialized(arena, r_extent_hooks); + if ((*r_extent_hooks)->merge(*r_extent_hooks, extent_base_get(a), + extent_size_get(a), extent_base_get(b), extent_size_get(b), + extent_committed_get(a), arena->ind)) return (true); /* diff --git a/src/extent_dss.c b/src/extent_dss.c index 0e34a440..9cc1d8f7 100644 --- a/src/extent_dss.c +++ b/src/extent_dss.c @@ -136,7 +136,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, dss_max = dss_next; malloc_mutex_unlock(tsdn, &dss_mtx); if (pad_size != 0) { - extent_hooks_t extent_hooks = + extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_dalloc_wrapper(tsdn, arena, &extent_hooks, pad); diff --git a/src/large.c b/src/large.c index 952d4644..5c7b4453 100644 --- a/src/large.c +++ b/src/large.c @@ -96,7 +96,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { arena_t *arena = extent_arena_get(extent); size_t oldusize = extent_usize_get(extent); - extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena); + extent_hooks_t *extent_hooks = extent_hooks_get(arena); size_t diff = extent_size_get(extent) - (usize + large_pad); assert(oldusize > usize); @@ -129,7 +129,7 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, arena_t *arena = extent_arena_get(extent); size_t oldusize = extent_usize_get(extent); bool is_zeroed_trail = false; - extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena); + extent_hooks_t *extent_hooks = extent_hooks_get(arena); size_t trailsize = usize - extent_usize_get(extent); extent_t *trail; diff --git a/test/integration/extent.c b/test/integration/extent.c index 15b96a00..78176711 100644 --- a/test/integration/extent.c +++ b/test/integration/extent.c @@ -4,8 +4,8 @@ const char *malloc_conf = "junk:false"; #endif -static extent_hooks_t orig_hooks; -static extent_hooks_t old_hooks; +static extent_hooks_t *orig_hooks; +static extent_hooks_t *old_hooks; static bool do_dalloc = true; static bool do_decommit; @@ -24,96 +24,111 @@ static bool did_merge; # define TRACE_HOOK(fmt, ...) #endif -void * -extent_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, - bool *commit, unsigned arena_ind) +static void * +extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, + size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { - TRACE_HOOK("%s(new_addr=%p, size=%zu, alignment=%zu, *zero=%s, " - "*commit=%s, arena_ind=%u)\n", __func__, new_addr, size, alignment, - *zero ? "true" : "false", *commit ? "true" : "false", arena_ind); + TRACE_HOOK("%s(extent_hooks=%p, new_addr=%p, size=%zu, alignment=%zu, " + "*zero=%s, *commit=%s, arena_ind=%u)\n", __func__, extent_hooks, + new_addr, size, alignment, *zero ? "true" : "false", *commit ? + "true" : "false", arena_ind); + assert(extent_hooks->alloc == extent_alloc); did_alloc = true; - return (old_hooks.alloc(new_addr, size, alignment, zero, commit, - arena_ind)); + return (old_hooks->alloc(old_hooks, new_addr, size, alignment, zero, + commit, arena_ind)); } -bool -extent_dalloc(void *addr, size_t size, bool committed, unsigned arena_ind) +static bool +extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, + bool committed, unsigned arena_ind) { - TRACE_HOOK("%s(addr=%p, size=%zu, committed=%s, arena_ind=%u)\n", - __func__, addr, size, committed ? "true" : "false", arena_ind); + TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, committed=%s, " + "arena_ind=%u)\n", __func__, extent_hooks, addr, size, committed ? + "true" : "false", arena_ind); + assert(extent_hooks->dalloc == extent_dalloc); did_dalloc = true; if (!do_dalloc) return (true); - return (old_hooks.dalloc(addr, size, committed, arena_ind)); + return (old_hooks->dalloc(old_hooks, addr, size, committed, arena_ind)); } -bool -extent_commit(void *addr, size_t size, size_t offset, size_t length, - unsigned arena_ind) +static bool +extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { bool err; - TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu, " - "arena_ind=%u)\n", __func__, addr, size, offset, length, + TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, " + "length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size, + offset, length, arena_ind); + assert(extent_hooks->commit == extent_commit); + err = old_hooks->commit(old_hooks, addr, size, offset, length, arena_ind); - err = old_hooks.commit(addr, size, offset, length, arena_ind); did_commit = !err; return (err); } -bool -extent_decommit(void *addr, size_t size, size_t offset, size_t length, - unsigned arena_ind) +static bool +extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { bool err; - TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu, " - "arena_ind=%u)\n", __func__, addr, size, offset, length, - arena_ind); + TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, " + "length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size, + offset, length, arena_ind); + assert(extent_hooks->decommit == extent_decommit); if (!do_decommit) return (true); - err = old_hooks.decommit(addr, size, offset, length, arena_ind); + err = old_hooks->decommit(old_hooks, addr, size, offset, length, + arena_ind); did_decommit = !err; return (err); } -bool -extent_purge(void *addr, size_t size, size_t offset, size_t length, - unsigned arena_ind) +static bool +extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { - TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu " - "arena_ind=%u)\n", __func__, addr, size, offset, length, - arena_ind); + TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, " + "length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size, + offset, length, arena_ind); + assert(extent_hooks->purge == extent_purge); did_purge = true; - return (old_hooks.purge(addr, size, offset, length, arena_ind)); -} - -bool -extent_split(void *addr, size_t size, size_t size_a, size_t size_b, - bool committed, unsigned arena_ind) -{ - - TRACE_HOOK("%s(addr=%p, size=%zu, size_a=%zu, size_b=%zu, " - "committed=%s, arena_ind=%u)\n", __func__, addr, size, size_a, - size_b, committed ? "true" : "false", arena_ind); - did_split = true; - return (old_hooks.split(addr, size, size_a, size_b, committed, + return (old_hooks->purge(old_hooks, addr, size, offset, length, arena_ind)); } -bool -extent_merge(void *addr_a, size_t size_a, void *addr_b, size_t size_b, - bool committed, unsigned arena_ind) +static bool +extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { - TRACE_HOOK("%s(addr_a=%p, size_a=%zu, addr_b=%p size_b=%zu, " - "committed=%s, arena_ind=%u)\n", __func__, addr_a, size_a, addr_b, - size_b, committed ? "true" : "false", arena_ind); + TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, size_a=%zu, " + "size_b=%zu, committed=%s, arena_ind=%u)\n", __func__, extent_hooks, + addr, size, size_a, size_b, committed ? "true" : "false", + arena_ind); + assert(extent_hooks->split == extent_split); + did_split = true; + return (old_hooks->split(old_hooks, addr, size, size_a, size_b, + committed, arena_ind)); +} + +static bool +extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + void *addr_b, size_t size_b, bool committed, unsigned arena_ind) +{ + + TRACE_HOOK("%s(extent_hooks=%p, addr_a=%p, size_a=%zu, addr_b=%p " + "size_b=%zu, committed=%s, arena_ind=%u)\n", __func__, extent_hooks, + addr_a, size_a, addr_b, size_b, committed ? "true" : "false", + arena_ind); + assert(extent_hooks->merge == extent_merge); did_merge = true; - return (old_hooks.merge(addr_a, size_a, addr_b, size_b, + return (old_hooks->merge(old_hooks, addr_a, size_a, addr_b, size_b, committed, arena_ind)); } @@ -125,7 +140,7 @@ TEST_BEGIN(test_extent) int flags; size_t hooks_mib[3], purge_mib[3]; size_t hooks_miblen, purge_miblen; - extent_hooks_t new_hooks = { + extent_hooks_t hooks = { extent_alloc, extent_dalloc, extent_commit, @@ -134,6 +149,7 @@ TEST_BEGIN(test_extent) extent_split, extent_merge }; + extent_hooks_t *new_hooks = &hooks; bool xallocx_success_a, xallocx_success_b, xallocx_success_c; sz = sizeof(unsigned); @@ -146,21 +162,21 @@ TEST_BEGIN(test_extent) assert_d_eq(mallctlnametomib("arena.0.extent_hooks", hooks_mib, &hooks_miblen), 0, "Unexpected mallctlnametomib() failure"); hooks_mib[1] = (size_t)arena_ind; - old_size = sizeof(extent_hooks_t); - new_size = sizeof(extent_hooks_t); + old_size = sizeof(extent_hooks_t *); + new_size = sizeof(extent_hooks_t *); assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size, &new_hooks, new_size), 0, "Unexpected extent_hooks error"); orig_hooks = old_hooks; - assert_ptr_ne(old_hooks.alloc, extent_alloc, "Unexpected alloc error"); - assert_ptr_ne(old_hooks.dalloc, extent_dalloc, + assert_ptr_ne(old_hooks->alloc, extent_alloc, "Unexpected alloc error"); + assert_ptr_ne(old_hooks->dalloc, extent_dalloc, "Unexpected dalloc error"); - assert_ptr_ne(old_hooks.commit, extent_commit, + assert_ptr_ne(old_hooks->commit, extent_commit, "Unexpected commit error"); - assert_ptr_ne(old_hooks.decommit, extent_decommit, + assert_ptr_ne(old_hooks->decommit, extent_decommit, "Unexpected decommit error"); - assert_ptr_ne(old_hooks.purge, extent_purge, "Unexpected purge error"); - assert_ptr_ne(old_hooks.split, extent_split, "Unexpected split error"); - assert_ptr_ne(old_hooks.merge, extent_merge, "Unexpected merge error"); + assert_ptr_ne(old_hooks->purge, extent_purge, "Unexpected purge error"); + assert_ptr_ne(old_hooks->split, extent_split, "Unexpected split error"); + assert_ptr_ne(old_hooks->merge, extent_merge, "Unexpected merge error"); /* Get large size classes. */ sz = sizeof(size_t); @@ -228,19 +244,20 @@ TEST_BEGIN(test_extent) &old_hooks, new_size), 0, "Unexpected extent_hooks error"); assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size, NULL, 0), 0, "Unexpected extent_hooks error"); - assert_ptr_eq(old_hooks.alloc, orig_hooks.alloc, + assert_ptr_eq(old_hooks, orig_hooks, "Unexpected hooks error"); + assert_ptr_eq(old_hooks->alloc, orig_hooks->alloc, "Unexpected alloc error"); - assert_ptr_eq(old_hooks.dalloc, orig_hooks.dalloc, + assert_ptr_eq(old_hooks->dalloc, orig_hooks->dalloc, "Unexpected dalloc error"); - assert_ptr_eq(old_hooks.commit, orig_hooks.commit, + assert_ptr_eq(old_hooks->commit, orig_hooks->commit, "Unexpected commit error"); - assert_ptr_eq(old_hooks.decommit, orig_hooks.decommit, + assert_ptr_eq(old_hooks->decommit, orig_hooks->decommit, "Unexpected decommit error"); - assert_ptr_eq(old_hooks.purge, orig_hooks.purge, + assert_ptr_eq(old_hooks->purge, orig_hooks->purge, "Unexpected purge error"); - assert_ptr_eq(old_hooks.split, orig_hooks.split, + assert_ptr_eq(old_hooks->split, orig_hooks->split, "Unexpected split error"); - assert_ptr_eq(old_hooks.merge, orig_hooks.merge, + assert_ptr_eq(old_hooks->merge, orig_hooks->merge, "Unexpected merge error"); } TEST_END