From 368baa42ef76f1dd44950b5929dc5697c0ac7add Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Tue, 3 Dec 2019 12:59:46 -0800 Subject: [PATCH] Extent -> Ehooks: Move purge_lazy hook. --- include/jemalloc/internal/ehooks.h | 22 +++++++++++++++---- src/base.c | 3 ++- src/ehooks.c | 18 +++++++++++++++ src/extent.c | 35 +++--------------------------- 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/include/jemalloc/internal/ehooks.h b/include/jemalloc/internal/ehooks.h index e9bdca37..c234ccdb 100644 --- a/include/jemalloc/internal/ehooks.h +++ b/include/jemalloc/internal/ehooks.h @@ -29,6 +29,11 @@ bool ehooks_default_commit(extent_hooks_t *extent_hooks, void *addr, size_t size bool ehooks_default_decommit_impl(void *addr, size_t offset, size_t length); bool ehooks_default_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); +#ifdef PAGES_CAN_PURGE_LAZY +bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length); +bool ehooks_default_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind); +#endif static inline void ehooks_pre_reentrancy(tsdn_t *tsdn) { @@ -168,14 +173,23 @@ ehooks_decommit(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, } static inline bool -ehooks_purge_lazy(ehooks_t *ehooks, void *addr, size_t size, size_t offset, - size_t length, unsigned arena_ind) { +ehooks_purge_lazy(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); +#ifdef PAGES_CAN_PURGE_LAZY + if (extent_hooks == &extent_hooks_default) { + return ehooks_default_purge_lazy_impl(addr, offset, length); + } +#endif if (extent_hooks->purge_lazy == NULL) { return true; + } else { + ehooks_pre_reentrancy(tsdn); + bool err = extent_hooks->purge_lazy(extent_hooks, addr, size, + offset, length, arena_ind); + ehooks_post_reentrancy(tsdn); + return err; } - return extent_hooks->purge_lazy(extent_hooks, addr, size, offset, - length, arena_ind); } static inline bool diff --git a/src/base.c b/src/base.c index 6b88b238..48a8c6ab 100644 --- a/src/base.c +++ b/src/base.c @@ -90,7 +90,8 @@ base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr, if (!ehooks_purge_forced(ehooks, addr, size, 0, size, ind)) { goto label_post_reentrancy; } - if (!ehooks_purge_lazy(ehooks, addr, size, 0, size, ind)) { + if (!ehooks_purge_lazy(tsdn, ehooks, addr, size, 0, size, + ind)) { goto label_post_reentrancy; } /* Nothing worked. That's the application's problem. */ diff --git a/src/ehooks.c b/src/ehooks.c index cb02377c..ae0e980c 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -126,3 +126,21 @@ ehooks_default_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind) { return ehooks_default_decommit_impl(addr, offset, length); } + +#ifdef PAGES_CAN_PURGE_LAZY +bool +ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length) { + return pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset), + length); +} + +bool +ehooks_default_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { + assert(addr != NULL); + assert((offset & PAGE_MASK) == 0); + assert(length != 0); + assert((length & PAGE_MASK) == 0); + return ehooks_default_purge_lazy_impl(addr, offset, length); +} +#endif diff --git a/src/extent.c b/src/extent.c index cb010643..f3fbe95f 100644 --- a/src/extent.c +++ b/src/extent.c @@ -21,10 +21,6 @@ size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT; static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, bool growing_retained); -#ifdef PAGES_CAN_PURGE_LAZY -static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, - size_t size, size_t offset, size_t length, unsigned arena_ind); -#endif static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, bool growing_retained); @@ -55,7 +51,7 @@ const extent_hooks_t extent_hooks_default = { ehooks_default_commit, ehooks_default_decommit, #ifdef PAGES_CAN_PURGE_LAZY - extent_purge_lazy_default, + ehooks_default_purge_lazy, #else NULL, #endif @@ -1390,7 +1386,7 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, arena_ind_get(arena))) { zeroed = true; } else if (extent_state_get(extent) == extent_state_muzzy || - !ehooks_purge_lazy(ehooks, extent_base_get(extent), + !ehooks_purge_lazy(tsdn, ehooks, extent_base_get(extent), extent_size_get(extent), 0, extent_size_get(extent), arena_ind_get(arena))) { zeroed = false; @@ -1461,38 +1457,13 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, return err; } -#ifdef PAGES_CAN_PURGE_LAZY -static bool -extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, - size_t offset, size_t length, unsigned arena_ind) { - assert(addr != NULL); - assert((offset & PAGE_MASK) == 0); - assert(length != 0); - assert((length & PAGE_MASK) == 0); - - return pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset), - length); -} -#endif - static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); - - if (ehooks_purge_lazy_will_fail(ehooks)) { - return true; - } - if (!ehooks_are_default(ehooks)) { - extent_hook_pre_reentrancy(tsdn, arena); - } - bool err = ehooks_purge_lazy(ehooks, extent_base_get(extent), + bool err = ehooks_purge_lazy(tsdn, ehooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena)); - if (!ehooks_are_default(ehooks)) { - extent_hook_post_reentrancy(tsdn); - } - return err; }