From a5b42a1a10048d9562d59e494c9e2cf3ab6943ba Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Tue, 3 Dec 2019 13:11:54 -0800 Subject: [PATCH] Extent -> Ehooks: Move purge_forced hook. --- include/jemalloc/internal/ehooks.h | 22 +++++++++++++++---- src/base.c | 3 ++- src/ehooks.c | 18 ++++++++++++++++ src/extent.c | 34 +++--------------------------- 4 files changed, 41 insertions(+), 36 deletions(-) diff --git a/include/jemalloc/internal/ehooks.h b/include/jemalloc/internal/ehooks.h index c234ccdb..ae5ef663 100644 --- a/include/jemalloc/internal/ehooks.h +++ b/include/jemalloc/internal/ehooks.h @@ -34,6 +34,11 @@ bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length); bool ehooks_default_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); #endif +#ifdef PAGES_CAN_PURGE_FORCED +bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length); +bool ehooks_default_purge_forced(extent_hooks_t *extent_hooks, void *addr, + size_t size, size_t offset, size_t length, unsigned arena_ind); +#endif static inline void ehooks_pre_reentrancy(tsdn_t *tsdn) { @@ -193,14 +198,23 @@ ehooks_purge_lazy(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, } static inline bool -ehooks_purge_forced(ehooks_t *ehooks, void *addr, size_t size, size_t offset, - size_t length, unsigned arena_ind) { +ehooks_purge_forced(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); +#ifdef PAGES_CAN_PURGE_FORCED + if (extent_hooks == &extent_hooks_default) { + return ehooks_default_purge_forced_impl(addr, offset, length); + } +#endif if (extent_hooks->purge_forced == NULL) { return true; + } else { + ehooks_pre_reentrancy(tsdn); + bool err = extent_hooks->purge_forced(extent_hooks, addr, size, + offset, length, arena_ind); + ehooks_post_reentrancy(tsdn); + return err; } - return extent_hooks->purge_forced(extent_hooks, addr, size, offset, - length, arena_ind); } static inline bool diff --git a/src/base.c b/src/base.c index 48a8c6ab..92d9bc1e 100644 --- a/src/base.c +++ b/src/base.c @@ -87,7 +87,8 @@ base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr, if (!ehooks_decommit(tsdn, ehooks, addr, size, 0, size, ind)) { goto label_post_reentrancy; } - if (!ehooks_purge_forced(ehooks, addr, size, 0, size, ind)) { + if (!ehooks_purge_forced(tsdn, ehooks, addr, size, 0, size, + ind)) { goto label_post_reentrancy; } if (!ehooks_purge_lazy(tsdn, ehooks, addr, size, 0, size, diff --git a/src/ehooks.c b/src/ehooks.c index ae0e980c..67ca2381 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -144,3 +144,21 @@ ehooks_default_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, return ehooks_default_purge_lazy_impl(addr, offset, length); } #endif + +#ifdef PAGES_CAN_PURGE_FORCED +bool +ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length) { + return pages_purge_forced((void *)((uintptr_t)addr + + (uintptr_t)offset), length); +} + +bool +ehooks_default_purge_forced(extent_hooks_t *extent_hooks, void *addr, + size_t size, size_t offset, size_t length, unsigned arena_ind) { + assert(addr != NULL); + assert((offset & PAGE_MASK) == 0); + assert(length != 0); + assert((length & PAGE_MASK) == 0); + return ehooks_default_purge_forced_impl(addr, offset, length); +} +#endif diff --git a/src/extent.c b/src/extent.c index f3fbe95f..f4f37976 100644 --- a/src/extent.c +++ b/src/extent.c @@ -24,10 +24,6 @@ static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, bool growing_retained); -#ifdef PAGES_CAN_PURGE_FORCED -static bool extent_purge_forced_default(extent_hooks_t *extent_hooks, - void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); -#endif static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, bool growing_retained); @@ -56,7 +52,7 @@ const extent_hooks_t extent_hooks_default = { NULL, #endif #ifdef PAGES_CAN_PURGE_FORCED - extent_purge_forced_default, + ehooks_default_purge_forced, #else NULL, #endif @@ -1381,7 +1377,7 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, } else if (!extent_decommit_wrapper(tsdn, arena, ehooks, extent, 0, extent_size_get(extent))) { zeroed = true; - } else if (!ehooks_purge_forced(ehooks, extent_base_get(extent), + } else if (!ehooks_purge_forced(tsdn, ehooks, extent_base_get(extent), extent_size_get(extent), 0, extent_size_get(extent), arena_ind_get(arena))) { zeroed = true; @@ -1474,37 +1470,13 @@ extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, length, false); } -#ifdef PAGES_CAN_PURGE_FORCED -static bool -extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr, - size_t size, size_t offset, size_t length, unsigned arena_ind) { - assert(addr != NULL); - assert((offset & PAGE_MASK) == 0); - assert(length != 0); - assert((length & PAGE_MASK) == 0); - - return pages_purge_forced((void *)((uintptr_t)addr + - (uintptr_t)offset), length); -} -#endif - static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); - - if (ehooks_purge_forced_will_fail(ehooks)) { - return true; - } - if (!ehooks_are_default(ehooks)) { - extent_hook_pre_reentrancy(tsdn, arena); - } - bool err = ehooks_purge_forced(ehooks, extent_base_get(extent), + bool err = ehooks_purge_forced(tsdn, ehooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena)); - if (!ehooks_are_default(ehooks)) { - extent_hook_post_reentrancy(tsdn); - } return err; }