From a6e86810d83aba0d94d0f6423ed09e8e6e0909fa Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 3 Dec 2016 15:38:25 -0800 Subject: [PATCH] Refactor purging and splitting/merging. Split purging into lazy and forced variants. Use the forced variant for zeroing dss. Add support for NULL function pointers as an opt-out mechanism for the dalloc, commit, decommit, purge_lazy, purge_forced, split, and merge fields of extent_hooks_t. Add short-circuiting checks in large_ralloc_no_move_{shrink,expand}() so that no attempt is made if splitting/merging is not supported. This resolves #268. --- doc/jemalloc.xml.in | 29 ++-- include/jemalloc/internal/arena.h | 4 +- include/jemalloc/internal/extent.h | 5 +- include/jemalloc/internal/pages.h | 35 ++++- include/jemalloc/internal/private_symbols.txt | 6 +- include/jemalloc/jemalloc_typedefs.h.in | 3 +- src/extent.c | 134 ++++++++++++++---- src/extent_dss.c | 14 +- src/large.c | 6 + src/pages.c | 43 +++--- test/integration/extent.c | 63 +++++--- 11 files changed, 258 insertions(+), 84 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 250a2a83..990aacf3 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1512,7 +1512,8 @@ struct extent_hooks_s { extent_dalloc_t *dalloc; extent_commit_t *commit; extent_decommit_t *decommit; - extent_purge_t *purge; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; extent_split_t *split; extent_merge_t *merge; };]]> @@ -1522,13 +1523,12 @@ struct extent_hooks_s { mapped committed memory, in the simplest case followed by deallocation. However, there are performance and platform reasons to retain extents for later reuse. Cleanup attempts cascade from deallocation to decommit - to purging, which gives the extent management functions opportunities to - reject the most permanent cleanup operations in favor of less permanent - (and often less costly) operations. The extent splitting and merging - operations can also be opted out of, but this is mainly intended to - support platforms on which virtual memory mappings provided by the - operating system kernel do not automatically coalesce and split, e.g. - Windows. + to lazy purging to forced purging, which gives the extent management + functions opportunities to reject the most permanent cleanup operations + in favor of less permanent (and often less costly) operations. All + operations except allocation can be universally opted out of by setting + the hook pointers to NULL, or selectively opted out + of by returning failure. typedef void *(extent_alloc_t) @@ -1634,21 +1634,24 @@ struct extent_hooks_s { typedef bool (extent_purge_t) extent_hooks_t *extent_hooks void *addr - size_tsize + size_t size size_t offset size_t length unsigned arena_ind An extent purge function conforms to the - extent_purge_t type and optionally discards physical pages + extent_purge_t type and discards physical pages within the virtual memory mapping associated with an extent at given addr and size at offset bytes, extending for length on behalf of arena - arena_ind, returning false if pages within the - purged virtual memory range will be zero-filled the next time they are - accessed. + arena_ind. A lazy extent purge function can + delay purging indefinitely and leave the pages within the purged virtual + memory range in an indeterminite state, whereas a forced extent purge + function immediately purges, and the pages within the virtual memory + range will be zero-filled the next time they are accessed. If the + function returns true, this indicates failure to purge. typedef bool (extent_split_t) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 6532b08a..a8c2976c 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -198,8 +198,8 @@ struct arena_s { /* * Current count of pages within unused extents that are potentially - * dirty, and for which madvise(... MADV_DONTNEED) has not been called. - * By tracking this, we can institute a limit on how much dirty unused + * dirty, and for which pages_purge_*() has not been called. By + * tracking this, we can institute a limit on how much dirty unused * memory is mapped for each arena. */ size_t ndirty; diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index d5690c08..33b85145 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -133,7 +133,10 @@ bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length); -bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, +bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, + size_t length); +bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length); extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h index 034a8aac..98e4f38a 100644 --- a/include/jemalloc/internal/pages.h +++ b/include/jemalloc/internal/pages.h @@ -24,6 +24,23 @@ #define HUGEPAGE_CEILING(s) \ (((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK) +/* PAGES_CAN_PURGE_LAZY is defined if lazy purging is supported. */ +#if defined(_WIN32) || defined(JEMALLOC_PURGE_MADVISE_FREE) +# define PAGES_CAN_PURGE_LAZY +#endif +/* + * PAGES_CAN_PURGE_FORCED is defined if forced purging is supported. + * + * The only supported way to hard-purge on Windows is to decommit and then + * re-commit, but doing so is racy, and if re-commit fails it's a pain to + * propagate the "poisoned" memory state. Since we typically decommit as the + * next step after purging on Windows anyway, there's no point in adding such + * complexity. + */ +#if !defined(_WIN32) && defined(JEMALLOC_PURGE_MADVISE_DONTNEED) +# define PAGES_CAN_PURGE_FORCED +#endif + #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS @@ -32,13 +49,29 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS +static const bool pages_can_purge_lazy = +#ifdef PAGES_CAN_PURGE_LAZY + true +#else + false +#endif + ; +static const bool pages_can_purge_forced = +#ifdef PAGES_CAN_PURGE_FORCED + true +#else + false +#endif + ; + void *pages_map(void *addr, size_t size, bool *commit); void pages_unmap(void *addr, size_t size); void *pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size, bool *commit); bool pages_commit(void *addr, size_t size); bool pages_decommit(void *addr, size_t size); -bool pages_purge(void *addr, size_t size); +bool pages_purge_lazy(void *addr, size_t size); +bool pages_purge_forced(void *addr, size_t size); bool pages_huge(void *addr, size_t size); bool pages_nohuge(void *addr, size_t size); void pages_boot(void); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 1facc928..7aa622fb 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -179,7 +179,8 @@ extent_merge_wrapper extent_past_get extent_prof_tctx_get extent_prof_tctx_set -extent_purge_wrapper +extent_purge_forced_wrapper +extent_purge_lazy_wrapper extent_retained_get extent_ring_insert extent_ring_remove @@ -327,7 +328,8 @@ pages_decommit pages_huge pages_map pages_nohuge -pages_purge +pages_purge_forced +pages_purge_lazy pages_trim pages_unmap pind2sz diff --git a/include/jemalloc/jemalloc_typedefs.h.in b/include/jemalloc/jemalloc_typedefs.h.in index 1049d7c7..91b5a8dc 100644 --- a/include/jemalloc/jemalloc_typedefs.h.in +++ b/include/jemalloc/jemalloc_typedefs.h.in @@ -61,7 +61,8 @@ struct extent_hooks_s { extent_dalloc_t *dalloc; extent_commit_t *commit; extent_decommit_t *decommit; - extent_purge_t *purge; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; extent_split_t *split; extent_merge_t *merge; }; diff --git a/src/extent.c b/src/extent.c index 586e8d33..827a9213 100644 --- a/src/extent.c +++ b/src/extent.c @@ -15,23 +15,47 @@ static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); static bool extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); -static bool extent_purge_default(extent_hooks_t *extent_hooks, void *addr, - size_t size, size_t offset, size_t length, unsigned arena_ind); +#ifdef PAGES_CAN_PURGE_LAZY +static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, + void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); +#endif +#ifdef PAGES_CAN_PURGE_FORCED +static bool extent_purge_forced_default(extent_hooks_t *extent_hooks, + void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); +#endif +#ifdef JEMALLOC_MAPS_COALESCE static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind); static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind); +#endif const extent_hooks_t extent_hooks_default = { extent_alloc_default, extent_dalloc_default, extent_commit_default, - extent_decommit_default, - extent_purge_default, + extent_decommit_default +#ifdef PAGES_CAN_PURGE_LAZY + , + extent_purge_lazy_default +#else + , + NULL +#endif +#ifdef PAGES_CAN_PURGE_FORCED + , + extent_purge_forced_default +#else + , + NULL +#endif +#ifdef JEMALLOC_MAPS_COALESCE + , extent_split_default, extent_merge_default +#endif }; /* Used exclusively for gdump triggering. */ @@ -395,8 +419,11 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, * that this is only a virtual memory leak. */ if (cache) { - extent_purge_wrapper(tsdn, arena, r_extent_hooks, extent, 0, - extent_size_get(extent)); + if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks, + extent, 0, extent_size_get(extent))) { + extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks, + extent, 0, extent_size_get(extent)); + } } extent_dalloc(tsdn, arena, extent); } @@ -1023,7 +1050,7 @@ void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { - bool err; + bool err, zeroed; assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); @@ -1041,9 +1068,10 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, err = extent_dalloc_default_impl(extent_base_get(extent), extent_size_get(extent)); } else { - err = (*r_extent_hooks)->dalloc(*r_extent_hooks, + err = ((*r_extent_hooks)->dalloc == NULL || + (*r_extent_hooks)->dalloc(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), - extent_committed_get(extent), arena->ind); + extent_committed_get(extent), arena->ind)); } if (!err) { @@ -1052,13 +1080,24 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, } extent_reregister(tsdn, extent); /* Try to decommit; purge if that fails. */ - if (extent_committed_get(extent)) { - extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, - 0, extent_size_get(extent)); - } - extent_zeroed_set(extent, !extent_committed_get(extent) || - !(*r_extent_hooks)->purge(*r_extent_hooks, extent_base_get(extent), - extent_size_get(extent), 0, extent_size_get(extent), arena->ind)); + if (!extent_committed_get(extent)) + zeroed = true; + else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, + 0, extent_size_get(extent))) + zeroed = true; + else if ((*r_extent_hooks)->purge_lazy != NULL && + !(*r_extent_hooks)->purge_lazy(*r_extent_hooks, + extent_base_get(extent), extent_size_get(extent), 0, + extent_size_get(extent), arena->ind)) + zeroed = false; + else if ((*r_extent_hooks)->purge_forced != NULL && + !(*r_extent_hooks)->purge_forced(*r_extent_hooks, + extent_base_get(extent), extent_size_get(extent), 0, + extent_size_get(extent), arena->ind)) + zeroed = true; + else + zeroed = false; + extent_zeroed_set(extent, zeroed); if (config_stats) arena->stats.retained += extent_size_get(extent); @@ -1088,9 +1127,9 @@ extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, bool err; extent_hooks_assure_initialized(arena, r_extent_hooks); - err = (*r_extent_hooks)->commit(*r_extent_hooks, - extent_base_get(extent), extent_size_get(extent), offset, length, - arena->ind); + err = ((*r_extent_hooks)->commit == NULL || + (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent), + extent_size_get(extent), offset, length, arena->ind)); extent_committed_set(extent, extent_committed_get(extent) || !err); return (err); } @@ -1115,15 +1154,17 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena, r_extent_hooks); - err = (*r_extent_hooks)->decommit(*r_extent_hooks, + err = ((*r_extent_hooks)->decommit == NULL || + (*r_extent_hooks)->decommit(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, - arena->ind); + arena->ind)); extent_committed_set(extent, extent_committed_get(extent) && err); return (err); } +#ifdef PAGES_CAN_PURGE_LAZY static bool -extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size, +extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind) { @@ -1133,22 +1174,55 @@ extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size, assert(length != 0); assert((length & PAGE_MASK) == 0); - return (pages_purge((void *)((uintptr_t)addr + (uintptr_t)offset), + return (pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset), length)); } +#endif bool -extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, +extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { extent_hooks_assure_initialized(arena, r_extent_hooks); - return ((*r_extent_hooks)->purge(*r_extent_hooks, + return ((*r_extent_hooks)->purge_lazy == NULL || + (*r_extent_hooks)->purge_lazy(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena->ind)); } +#ifdef PAGES_CAN_PURGE_FORCED +static bool +extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr, + size_t size, size_t offset, size_t length, unsigned arena_ind) +{ + + assert(extent_hooks == &extent_hooks_default); + assert(addr != NULL); + assert((offset & PAGE_MASK) == 0); + assert(length != 0); + assert((length & PAGE_MASK) == 0); + + return (pages_purge_forced((void *)((uintptr_t)addr + + (uintptr_t)offset), length)); +} +#endif + +bool +extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, + size_t length) +{ + + extent_hooks_assure_initialized(arena, r_extent_hooks); + return ((*r_extent_hooks)->purge_forced == NULL || + (*r_extent_hooks)->purge_forced(*r_extent_hooks, + extent_base_get(extent), extent_size_get(extent), offset, length, + arena->ind)); +} + +#ifdef JEMALLOC_MAPS_COALESCE static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind) @@ -1160,6 +1234,7 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, return (true); return (false); } +#endif extent_t * extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, @@ -1175,6 +1250,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena, r_extent_hooks); + if ((*r_extent_hooks)->split == NULL) + return (NULL); + trail = extent_alloc(tsdn, arena); if (trail == NULL) goto label_error_a; @@ -1237,6 +1315,7 @@ extent_merge_default_impl(void *addr_a, void *addr_b) return (false); } +#ifdef JEMALLOC_MAPS_COALESCE static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind) @@ -1246,6 +1325,7 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, return (extent_merge_default_impl(addr_a, addr_b)); } +#endif bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, @@ -1257,6 +1337,10 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; extent_hooks_assure_initialized(arena, r_extent_hooks); + + if ((*r_extent_hooks)->merge == NULL) + return (true); + if (*r_extent_hooks == &extent_hooks_default) { /* Call directly to propagate tsdn. */ err = extent_merge_default_impl(extent_base_get(a), diff --git a/src/extent_dss.c b/src/extent_dss.c index 1169d496..0f0c689b 100644 --- a/src/extent_dss.c +++ b/src/extent_dss.c @@ -168,10 +168,20 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, extent_dalloc_gap(tsdn, arena, gap); else extent_dalloc(tsdn, arena, gap); - if (*zero) - memset(ret, 0, size); if (!*commit) *commit = pages_decommit(ret, size); + if (*zero && *commit) { + extent_hooks_t *extent_hooks = + EXTENT_HOOKS_INITIALIZER; + extent_t extent; + + extent_init(&extent, arena, ret, size, + size, 0, true, false, true, false); + if (extent_purge_forced_wrapper(tsdn, + arena, &extent_hooks, &extent, 0, + size)) + memset(ret, 0, size); + } return (ret); } /* diff --git a/src/large.c b/src/large.c index 1bae9399..ec22e64c 100644 --- a/src/large.c +++ b/src/large.c @@ -110,6 +110,9 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) assert(oldusize > usize); + if (extent_hooks->split == NULL) + return (true); + /* Split excess pages. */ if (diff != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, @@ -142,6 +145,9 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, size_t trailsize = usize - extent_usize_get(extent); extent_t *trail; + if (extent_hooks->merge == NULL) + return (true); + if ((trail = arena_extent_cache_alloc(tsdn, arena, &extent_hooks, extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail)) == NULL) { diff --git a/src/pages.c b/src/pages.c index 8bef6fac..d5a0a21c 100644 --- a/src/pages.c +++ b/src/pages.c @@ -163,33 +163,34 @@ pages_decommit(void *addr, size_t size) } bool -pages_purge(void *addr, size_t size) +pages_purge_lazy(void *addr, size_t size) { - bool unzeroed; + + if (!pages_can_purge_lazy) + return (true); #ifdef _WIN32 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); - unzeroed = true; -#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \ - defined(JEMALLOC_PURGE_MADVISE_DONTNEED)) -# if defined(JEMALLOC_PURGE_MADVISE_FREE) -# define JEMALLOC_MADV_PURGE MADV_FREE -# define JEMALLOC_MADV_ZEROS false -# elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) -# define JEMALLOC_MADV_PURGE MADV_DONTNEED -# define JEMALLOC_MADV_ZEROS true -# else -# error No madvise(2) flag defined for purging unused dirty pages -# endif - int err = madvise(addr, size, JEMALLOC_MADV_PURGE); - unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); -# undef JEMALLOC_MADV_PURGE -# undef JEMALLOC_MADV_ZEROS +#elif defined(JEMALLOC_PURGE_MADVISE_FREE) + madvise(addr, size, MADV_FREE); #else - /* Last resort no-op. */ - unzeroed = true; + not_reached(); +#endif + return (false); +} + +bool +pages_purge_forced(void *addr, size_t size) +{ + + if (!pages_can_purge_forced) + return (true); + +#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED) + return (madvise(addr, size, MADV_DONTNEED) != 0); +#else + not_reached(); #endif - return (unzeroed); } bool diff --git a/test/integration/extent.c b/test/integration/extent.c index 2af20ce2..b0fc52d6 100644 --- a/test/integration/extent.c +++ b/test/integration/extent.c @@ -13,7 +13,9 @@ static bool extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); static bool extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); -static bool extent_purge(extent_hooks_t *extent_hooks, void *addr, +static bool extent_purge_lazy(extent_hooks_t *extent_hooks, void *addr, + size_t size, size_t offset, size_t length, unsigned arena_ind); +static bool extent_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); static bool extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, @@ -27,7 +29,8 @@ static extent_hooks_t hooks = { extent_dalloc, extent_commit, extent_decommit, - extent_purge, + extent_purge_lazy, + extent_purge_forced, extent_split, extent_merge }; @@ -42,7 +45,8 @@ static bool did_alloc; static bool did_dalloc; static bool did_commit; static bool did_decommit; -static bool did_purge; +static bool did_purge_lazy; +static bool did_purge_forced; static bool tried_split; static bool did_split; static bool did_merge; @@ -129,7 +133,7 @@ extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, } static bool -extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size, +extent_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind) { @@ -138,9 +142,29 @@ extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size, offset, length, arena_ind); assert_ptr_eq(extent_hooks, new_hooks, "extent_hooks should be same as pointer used to set hooks"); - assert_ptr_eq(extent_hooks->purge, extent_purge, "Wrong hook function"); - did_purge = true; - return (old_hooks->purge(old_hooks, addr, size, offset, length, + assert_ptr_eq(extent_hooks->purge_lazy, extent_purge_lazy, + "Wrong hook function"); + did_purge_lazy = true; + return (old_hooks->purge_lazy == NULL || + old_hooks->purge_lazy(old_hooks, addr, size, offset, length, + arena_ind)); +} + +static bool +extent_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) +{ + + TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, " + "length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size, + offset, length, arena_ind); + assert_ptr_eq(extent_hooks, new_hooks, + "extent_hooks should be same as pointer used to set hooks"); + assert_ptr_eq(extent_hooks->purge_forced, extent_purge_forced, + "Wrong hook function"); + did_purge_forced = true; + return (old_hooks->purge_forced == NULL || + old_hooks->purge_forced(old_hooks, addr, size, offset, length, arena_ind)); } @@ -158,8 +182,8 @@ extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, "extent_hooks should be same as pointer used to set hooks"); assert_ptr_eq(extent_hooks->split, extent_split, "Wrong hook function"); tried_split = true; - err = old_hooks->split(old_hooks, addr, size, size_a, size_b, committed, - arena_ind); + err = (old_hooks->split == NULL || old_hooks->split(old_hooks, addr, + size, size_a, size_b, committed, arena_ind)); did_split = !err; return (err); } @@ -177,8 +201,8 @@ extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, assert_ptr_eq(extent_hooks, new_hooks, "extent_hooks should be same as pointer used to set hooks"); assert_ptr_eq(extent_hooks->merge, extent_merge, "Wrong hook function"); - err = old_hooks->merge(old_hooks, addr_a, size_a, addr_b, size_b, - committed, arena_ind); + err = (old_hooks->merge == NULL || old_hooks->merge(old_hooks, addr_a, + size_a, addr_b, size_b, committed, arena_ind)); did_merge = !err; return (err); } @@ -216,7 +240,10 @@ TEST_BEGIN(test_extent) "Unexpected commit error"); assert_ptr_ne(old_hooks->decommit, extent_decommit, "Unexpected decommit error"); - assert_ptr_ne(old_hooks->purge, extent_purge, "Unexpected purge error"); + assert_ptr_ne(old_hooks->purge_lazy, extent_purge_lazy, + "Unexpected purge_lazy error"); + assert_ptr_ne(old_hooks->purge_forced, extent_purge_forced, + "Unexpected purge_forced error"); assert_ptr_ne(old_hooks->split, extent_split, "Unexpected split error"); assert_ptr_ne(old_hooks->merge, extent_merge, "Unexpected merge error"); @@ -240,7 +267,8 @@ TEST_BEGIN(test_extent) assert_ptr_not_null(p, "Unexpected mallocx() error"); did_dalloc = false; did_decommit = false; - did_purge = false; + did_purge_lazy = false; + did_purge_forced = false; tried_split = false; did_split = false; xallocx_success_a = (xallocx(p, large0, 0, flags) == large0); @@ -249,7 +277,8 @@ TEST_BEGIN(test_extent) if (xallocx_success_a) { assert_true(did_dalloc, "Expected dalloc"); assert_false(did_decommit, "Unexpected decommit"); - assert_true(did_purge, "Expected purge"); + assert_true(did_purge_lazy || did_purge_forced, + "Expected purge"); } assert_true(tried_split, "Expected split"); dallocx(p, flags); @@ -300,8 +329,10 @@ TEST_BEGIN(test_extent) "Unexpected commit error"); assert_ptr_eq(old_hooks->decommit, orig_hooks->decommit, "Unexpected decommit error"); - assert_ptr_eq(old_hooks->purge, orig_hooks->purge, - "Unexpected purge error"); + assert_ptr_eq(old_hooks->purge_lazy, orig_hooks->purge_lazy, + "Unexpected purge_lazy error"); + assert_ptr_eq(old_hooks->purge_forced, orig_hooks->purge_forced, + "Unexpected purge_forced error"); assert_ptr_eq(old_hooks->split, orig_hooks->split, "Unexpected split error"); assert_ptr_eq(old_hooks->merge, orig_hooks->merge,