Refactor purging and splitting/merging.

Split purging into lazy and forced variants.  Use the forced variant for
zeroing dss.

Add support for NULL function pointers as an opt-out mechanism for the
dalloc, commit, decommit, purge_lazy, purge_forced, split, and merge
fields of extent_hooks_t.

Add short-circuiting checks in large_ralloc_no_move_{shrink,expand}() so
that no attempt is made if splitting/merging is not supported.

This resolves #268.

doc/jemalloc.xml.in

@@ -1512,7 +1512,8 @@ struct extent_hooks_s {
 	extent_dalloc_t		*dalloc;
 	extent_commit_t		*commit;
 	extent_decommit_t	*decommit;
-	extent_purge_t		*purge;
+	extent_purge_t		*purge_lazy;
+	extent_purge_t		*purge_forced;
 	extent_split_t		*split;
 	extent_merge_t		*merge;
 };]]></programlisting>
@@ -1522,13 +1523,12 @@ struct extent_hooks_s {
         mapped committed memory, in the simplest case followed by deallocation.
         However, there are performance and platform reasons to retain extents
         for later reuse.  Cleanup attempts cascade from deallocation to decommit
-        to purging, which gives the extent management functions opportunities to
+        to lazy purging to forced purging, which gives the extent management
-        reject the most permanent cleanup operations in favor of less permanent
+        functions opportunities to reject the most permanent cleanup operations
-        (and often less costly) operations.  The extent splitting and merging
+        in favor of less permanent (and often less costly) operations.  All
-        operations can also be opted out of, but this is mainly intended to
+        operations except allocation can be universally opted out of by setting
-        support platforms on which virtual memory mappings provided by the
+        the hook pointers to <constant>NULL</constant>, or selectively opted out
-        operating system kernel do not automatically coalesce and split, e.g.
+        of by returning failure.</para>
-        Windows.</para>
 
         <funcsynopsis><funcprototype>
           <funcdef>typedef void *<function>(extent_alloc_t)</function></funcdef>
@@ -1634,21 +1634,24 @@ struct extent_hooks_s {
           <funcdef>typedef bool <function>(extent_purge_t)</function></funcdef>
           <paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
           <paramdef>void *<parameter>addr</parameter></paramdef>
-          <paramdef>size_t<parameter>size</parameter></paramdef>
+          <paramdef>size_t <parameter>size</parameter></paramdef>
           <paramdef>size_t <parameter>offset</parameter></paramdef>
           <paramdef>size_t <parameter>length</parameter></paramdef>
           <paramdef>unsigned <parameter>arena_ind</parameter></paramdef>
         </funcprototype></funcsynopsis>
         <literallayout></literallayout>
         <para>An extent purge function conforms to the
-        <type>extent_purge_t</type> type and optionally discards physical pages
+        <type>extent_purge_t</type> type and discards physical pages
         within the virtual memory mapping associated with an extent at given
         <parameter>addr</parameter> and <parameter>size</parameter> at
         <parameter>offset</parameter> bytes, extending for
         <parameter>length</parameter> on behalf of arena
-        <parameter>arena_ind</parameter>, returning false if pages within the
+        <parameter>arena_ind</parameter>.  A lazy extent purge function can
-        purged virtual memory range will be zero-filled the next time they are
+        delay purging indefinitely and leave the pages within the purged virtual
-        accessed.</para>
+        memory range in an indeterminite state, whereas a forced extent purge
+        function immediately purges, and the pages within the virtual memory
+        range will be zero-filled the next time they are accessed.  If the
+        function returns true, this indicates failure to purge.</para>
 
         <funcsynopsis><funcprototype>
           <funcdef>typedef bool <function>(extent_split_t)</function></funcdef>

include/jemalloc/internal/arena.h

@@ -198,8 +198,8 @@ struct arena_s {
 
 	/*
 	 * Current count of pages within unused extents that are potentially
-	 * dirty, and for which madvise(... MADV_DONTNEED) has not been called.
+	 * dirty, and for which pages_purge_*() has not been called.  By
-	 * By tracking this, we can institute a limit on how much dirty unused
+	 * tracking this, we can institute a limit on how much dirty unused
 	 * memory is mapped for each arena.
 	 */
 	size_t			ndirty;

include/jemalloc/internal/extent.h

@@ -133,7 +133,10 @@ bool	extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
 bool	extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
     size_t length);
-bool	extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
+bool	extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length);
+bool	extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
     size_t length);
 extent_t	*extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,

include/jemalloc/internal/pages.h

@@ -24,6 +24,23 @@
 #define	HUGEPAGE_CEILING(s)						\
 	(((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK)
 
+/* PAGES_CAN_PURGE_LAZY is defined if lazy purging is supported. */
+#if defined(_WIN32) || defined(JEMALLOC_PURGE_MADVISE_FREE)
+#  define PAGES_CAN_PURGE_LAZY
+#endif
+/*
+ * PAGES_CAN_PURGE_FORCED is defined if forced purging is supported.
+ *
+ * The only supported way to hard-purge on Windows is to decommit and then
+ * re-commit, but doing so is racy, and if re-commit fails it's a pain to
+ * propagate the "poisoned" memory state.  Since we typically decommit as the
+ * next step after purging on Windows anyway, there's no point in adding such
+ * complexity.
+ */
+#if !defined(_WIN32) && defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
+#  define PAGES_CAN_PURGE_FORCED
+#endif
+
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
@@ -32,13 +49,29 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
+static const bool pages_can_purge_lazy =
+#ifdef PAGES_CAN_PURGE_LAZY
+    true
+#else
+    false
+#endif
+    ;
+static const bool pages_can_purge_forced =
+#ifdef PAGES_CAN_PURGE_FORCED
+    true
+#else
+    false
+#endif
+    ;
+
 void	*pages_map(void *addr, size_t size, bool *commit);
 void	pages_unmap(void *addr, size_t size);
 void	*pages_trim(void *addr, size_t alloc_size, size_t leadsize,
     size_t size, bool *commit);
 bool	pages_commit(void *addr, size_t size);
 bool	pages_decommit(void *addr, size_t size);
-bool	pages_purge(void *addr, size_t size);
+bool	pages_purge_lazy(void *addr, size_t size);
+bool	pages_purge_forced(void *addr, size_t size);
 bool	pages_huge(void *addr, size_t size);
 bool	pages_nohuge(void *addr, size_t size);
 void	pages_boot(void);

include/jemalloc/internal/private_symbols.txt

@@ -179,7 +179,8 @@ extent_merge_wrapper
 extent_past_get
 extent_prof_tctx_get
 extent_prof_tctx_set
-extent_purge_wrapper
+extent_purge_forced_wrapper
+extent_purge_lazy_wrapper
 extent_retained_get
 extent_ring_insert
 extent_ring_remove
@@ -327,7 +328,8 @@ pages_decommit
 pages_huge
 pages_map
 pages_nohuge
-pages_purge
+pages_purge_forced
+pages_purge_lazy
 pages_trim
 pages_unmap
 pind2sz

include/jemalloc/jemalloc_typedefs.h.in

@@ -61,7 +61,8 @@ struct extent_hooks_s {
 	extent_dalloc_t		*dalloc;
 	extent_commit_t		*commit;
 	extent_decommit_t	*decommit;
-	extent_purge_t		*purge;
+	extent_purge_t		*purge_lazy;
+	extent_purge_t		*purge_forced;
 	extent_split_t		*split;
 	extent_merge_t		*merge;
 };

src/extent.c

@@ -15,23 +15,47 @@ static bool	extent_commit_default(extent_hooks_t *extent_hooks, void *addr,
     size_t size, size_t offset, size_t length, unsigned arena_ind);
 static bool	extent_decommit_default(extent_hooks_t *extent_hooks,
     void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
-static bool	extent_purge_default(extent_hooks_t *extent_hooks, void *addr,
+#ifdef PAGES_CAN_PURGE_LAZY
-    size_t size, size_t offset, size_t length, unsigned arena_ind);
+static bool	extent_purge_lazy_default(extent_hooks_t *extent_hooks,
+    void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
+#endif
+#ifdef PAGES_CAN_PURGE_FORCED
+static bool	extent_purge_forced_default(extent_hooks_t *extent_hooks,
+    void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
+#endif
+#ifdef JEMALLOC_MAPS_COALESCE
 static bool	extent_split_default(extent_hooks_t *extent_hooks, void *addr,
     size_t size, size_t size_a, size_t size_b, bool committed,
     unsigned arena_ind);
 static bool	extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a,
     size_t size_a, void *addr_b, size_t size_b, bool committed,
     unsigned arena_ind);
+#endif
 
 const extent_hooks_t	extent_hooks_default = {
 	extent_alloc_default,
 	extent_dalloc_default,
 	extent_commit_default,
-	extent_decommit_default,
+	extent_decommit_default
-	extent_purge_default,
+#ifdef PAGES_CAN_PURGE_LAZY
+	,
+	extent_purge_lazy_default
+#else
+	,
+	NULL
+#endif
+#ifdef PAGES_CAN_PURGE_FORCED
+	,
+	extent_purge_forced_default
+#else
+	,
+	NULL
+#endif
+#ifdef JEMALLOC_MAPS_COALESCE
+	,
 	extent_split_default,
 	extent_merge_default
+#endif
 };
 
 /* Used exclusively for gdump triggering. */
@@ -395,8 +419,11 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 	 * that this is only a virtual memory leak.
 	 */
 	if (cache) {
-		extent_purge_wrapper(tsdn, arena, r_extent_hooks, extent, 0,
+		if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks,
-		    extent_size_get(extent));
+		    extent, 0, extent_size_get(extent))) {
+			extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks,
+			    extent, 0, extent_size_get(extent));
+		}
 	}
 	extent_dalloc(tsdn, arena, extent);
 }
@@ -1023,7 +1050,7 @@ void
 extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent)
 {
-	bool err;
+	bool err, zeroed;
 
 	assert(extent_base_get(extent) != NULL);
 	assert(extent_size_get(extent) != 0);
@@ -1041,9 +1068,10 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
 		err = extent_dalloc_default_impl(extent_base_get(extent),
 		    extent_size_get(extent));
 	} else {
-		err = (*r_extent_hooks)->dalloc(*r_extent_hooks,
+		err = ((*r_extent_hooks)->dalloc == NULL ||
+		    (*r_extent_hooks)->dalloc(*r_extent_hooks,
 		    extent_base_get(extent), extent_size_get(extent),
-		    extent_committed_get(extent), arena->ind);
+		    extent_committed_get(extent), arena->ind));
 	}
 
 	if (!err) {
@@ -1052,13 +1080,24 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
 	}
 	extent_reregister(tsdn, extent);
 	/* Try to decommit; purge if that fails. */
-	if (extent_committed_get(extent)) {
+	if (!extent_committed_get(extent))
-		extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
+		zeroed = true;
-		    0, extent_size_get(extent));
+	else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
-	}
+	    0, extent_size_get(extent)))
-	extent_zeroed_set(extent, !extent_committed_get(extent) ||
+		zeroed = true;
-	    !(*r_extent_hooks)->purge(*r_extent_hooks, extent_base_get(extent),
+	else if ((*r_extent_hooks)->purge_lazy != NULL &&
-	    extent_size_get(extent), 0, extent_size_get(extent), arena->ind));
+	    !(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
+	    extent_base_get(extent), extent_size_get(extent), 0,
+	    extent_size_get(extent), arena->ind))
+		zeroed = false;
+	else if ((*r_extent_hooks)->purge_forced != NULL &&
+	    !(*r_extent_hooks)->purge_forced(*r_extent_hooks,
+	    extent_base_get(extent), extent_size_get(extent), 0,
+	    extent_size_get(extent), arena->ind))
+		zeroed = true;
+	else
+		zeroed = false;
+	extent_zeroed_set(extent, zeroed);
 
 	if (config_stats)
 		arena->stats.retained += extent_size_get(extent);
@@ -1088,9 +1127,9 @@ extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
 	bool err;
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
-	err = (*r_extent_hooks)->commit(*r_extent_hooks,
+	err = ((*r_extent_hooks)->commit == NULL ||
-	    extent_base_get(extent), extent_size_get(extent), offset, length,
+	    (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent),
-	    arena->ind);
+	    extent_size_get(extent), offset, length, arena->ind));
 	extent_committed_set(extent, extent_committed_get(extent) || !err);
 	return (err);
 }
@@ -1115,15 +1154,17 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
 
-	err = (*r_extent_hooks)->decommit(*r_extent_hooks,
+	err = ((*r_extent_hooks)->decommit == NULL ||
+	    (*r_extent_hooks)->decommit(*r_extent_hooks,
 	    extent_base_get(extent), extent_size_get(extent), offset, length,
-	    arena->ind);
+	    arena->ind));
 	extent_committed_set(extent, extent_committed_get(extent) && err);
 	return (err);
 }
 
+#ifdef PAGES_CAN_PURGE_LAZY
 static bool
-extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
+extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
     size_t offset, size_t length, unsigned arena_ind)
 {
 
@@ -1133,22 +1174,55 @@ extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
 	assert(length != 0);
 	assert((length & PAGE_MASK) == 0);
 
-	return (pages_purge((void *)((uintptr_t)addr + (uintptr_t)offset),
+	return (pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset),
 	    length));
 }
+#endif
 
 bool
-extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
+extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
     size_t length)
 {
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
-	return ((*r_extent_hooks)->purge(*r_extent_hooks,
+	return ((*r_extent_hooks)->purge_lazy == NULL ||
+	    (*r_extent_hooks)->purge_lazy(*r_extent_hooks,
 	    extent_base_get(extent), extent_size_get(extent), offset, length,
 	    arena->ind));
 }
 
+#ifdef PAGES_CAN_PURGE_FORCED
+static bool
+extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, size_t offset, size_t length, unsigned arena_ind)
+{
+
+	assert(extent_hooks == &extent_hooks_default);
+	assert(addr != NULL);
+	assert((offset & PAGE_MASK) == 0);
+	assert(length != 0);
+	assert((length & PAGE_MASK) == 0);
+
+	return (pages_purge_forced((void *)((uintptr_t)addr +
+	    (uintptr_t)offset), length));
+}
+#endif
+
+bool
+extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length)
+{
+
+	extent_hooks_assure_initialized(arena, r_extent_hooks);
+	return ((*r_extent_hooks)->purge_forced == NULL ||
+	    (*r_extent_hooks)->purge_forced(*r_extent_hooks,
+	    extent_base_get(extent), extent_size_get(extent), offset, length,
+	    arena->ind));
+}
+
+#ifdef JEMALLOC_MAPS_COALESCE
 static bool
 extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
     size_t size_a, size_t size_b, bool committed, unsigned arena_ind)
@@ -1160,6 +1234,7 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
 		return (true);
 	return (false);
 }
+#endif
 
 extent_t *
 extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
@@ -1175,6 +1250,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
 
+	if ((*r_extent_hooks)->split == NULL)
+		return (NULL);
+
 	trail = extent_alloc(tsdn, arena);
 	if (trail == NULL)
 		goto label_error_a;
@@ -1237,6 +1315,7 @@ extent_merge_default_impl(void *addr_a, void *addr_b)
 	return (false);
 }
 
+#ifdef JEMALLOC_MAPS_COALESCE
 static bool
 extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
     void *addr_b, size_t size_b, bool committed, unsigned arena_ind)
@@ -1246,6 +1325,7 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
 
 	return (extent_merge_default_impl(addr_a, addr_b));
 }
+#endif
 
 bool
 extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
@@ -1257,6 +1337,10 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
 	rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
 
 	extent_hooks_assure_initialized(arena, r_extent_hooks);
+
+	if ((*r_extent_hooks)->merge == NULL)
+		return (true);
+
 	if (*r_extent_hooks == &extent_hooks_default) {
 		/* Call directly to propagate tsdn. */
 		err = extent_merge_default_impl(extent_base_get(a),

src/extent_dss.c

@@ -168,10 +168,20 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
 					extent_dalloc_gap(tsdn, arena, gap);
 				else
 					extent_dalloc(tsdn, arena, gap);
-				if (*zero)
-					memset(ret, 0, size);
 				if (!*commit)
 					*commit = pages_decommit(ret, size);
+				if (*zero && *commit) {
+					extent_hooks_t *extent_hooks =
+					    EXTENT_HOOKS_INITIALIZER;
+					extent_t extent;
+
+					extent_init(&extent, arena, ret, size,
+					    size, 0, true, false, true, false);
+					if (extent_purge_forced_wrapper(tsdn,
+					    arena, &extent_hooks, &extent, 0,
+					    size))
+						memset(ret, 0, size);
+				}
 				return (ret);
 			}
 			/*

src/large.c

@@ -110,6 +110,9 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
 
 	assert(oldusize > usize);
 
+	if (extent_hooks->split == NULL)
+		return (true);
+
 	/* Split excess pages. */
 	if (diff != 0) {
 		extent_t *trail = extent_split_wrapper(tsdn, arena,
@@ -142,6 +145,9 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
 	size_t trailsize = usize - extent_usize_get(extent);
 	extent_t *trail;
 
+	if (extent_hooks->merge == NULL)
+		return (true);
+
 	if ((trail = arena_extent_cache_alloc(tsdn, arena, &extent_hooks,
 	    extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail)) ==
 	    NULL) {

src/pages.c

@@ -163,33 +163,34 @@ pages_decommit(void *addr, size_t size)
 }
 
 bool
-pages_purge(void *addr, size_t size)
+pages_purge_lazy(void *addr, size_t size)
 {
-	bool unzeroed;
+
+	if (!pages_can_purge_lazy)
+		return (true);
 
 #ifdef _WIN32
 	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
-	unzeroed = true;
+#elif defined(JEMALLOC_PURGE_MADVISE_FREE)
-#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \
+	madvise(addr, size, MADV_FREE);
-    defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
-#  if defined(JEMALLOC_PURGE_MADVISE_FREE)
-#    define JEMALLOC_MADV_PURGE MADV_FREE
-#    define JEMALLOC_MADV_ZEROS false
-#  elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
-#    define JEMALLOC_MADV_PURGE MADV_DONTNEED
-#    define JEMALLOC_MADV_ZEROS true
-#  else
-#    error No madvise(2) flag defined for purging unused dirty pages
-#  endif
-	int err = madvise(addr, size, JEMALLOC_MADV_PURGE);
-	unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0);
-#  undef JEMALLOC_MADV_PURGE
-#  undef JEMALLOC_MADV_ZEROS
 #else
-	/* Last resort no-op. */
+	not_reached();
-	unzeroed = true;
+#endif
+	return (false);
+}
+
+bool
+pages_purge_forced(void *addr, size_t size)
+{
+
+	if (!pages_can_purge_forced)
+		return (true);
+
+#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
+	return (madvise(addr, size, MADV_DONTNEED) != 0);
+#else
+	not_reached();
 #endif
-	return (unzeroed);
 }
 
 bool

test/integration/extent.c

@@ -13,7 +13,9 @@ static bool	extent_commit(extent_hooks_t *extent_hooks, void *addr,
     size_t size, size_t offset, size_t length, unsigned arena_ind);
 static bool	extent_decommit(extent_hooks_t *extent_hooks, void *addr,
     size_t size, size_t offset, size_t length, unsigned arena_ind);
-static bool	extent_purge(extent_hooks_t *extent_hooks, void *addr,
+static bool	extent_purge_lazy(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, size_t offset, size_t length, unsigned arena_ind);
+static bool	extent_purge_forced(extent_hooks_t *extent_hooks, void *addr,
     size_t size, size_t offset, size_t length, unsigned arena_ind);
 static bool	extent_split(extent_hooks_t *extent_hooks, void *addr,
     size_t size, size_t size_a, size_t size_b, bool committed,
@@ -27,7 +29,8 @@ static extent_hooks_t hooks = {
 	extent_dalloc,
 	extent_commit,
 	extent_decommit,
-	extent_purge,
+	extent_purge_lazy,
+	extent_purge_forced,
 	extent_split,
 	extent_merge
 };
@@ -42,7 +45,8 @@ static bool did_alloc;
 static bool did_dalloc;
 static bool did_commit;
 static bool did_decommit;
-static bool did_purge;
+static bool did_purge_lazy;
+static bool did_purge_forced;
 static bool tried_split;
 static bool did_split;
 static bool did_merge;
@@ -129,7 +133,7 @@ extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size,
 }
 
 static bool
-extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size,
+extent_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size,
     size_t offset, size_t length, unsigned arena_ind)
 {
 
@@ -138,9 +142,29 @@ extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size,
 	    offset, length, arena_ind);
 	assert_ptr_eq(extent_hooks, new_hooks,
 	    "extent_hooks should be same as pointer used to set hooks");
-	assert_ptr_eq(extent_hooks->purge, extent_purge, "Wrong hook function");
+	assert_ptr_eq(extent_hooks->purge_lazy, extent_purge_lazy,
-	did_purge = true;
+	    "Wrong hook function");
-	return (old_hooks->purge(old_hooks, addr, size, offset, length,
+	did_purge_lazy = true;
+	return (old_hooks->purge_lazy == NULL ||
+	    old_hooks->purge_lazy(old_hooks, addr, size, offset, length,
+	    arena_ind));
+}
+
+static bool
+extent_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    size_t offset, size_t length, unsigned arena_ind)
+{
+
+	TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
+	    "length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size,
+	    offset, length, arena_ind);
+	assert_ptr_eq(extent_hooks, new_hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->purge_forced, extent_purge_forced,
+	    "Wrong hook function");
+	did_purge_forced = true;
+	return (old_hooks->purge_forced == NULL ||
+	    old_hooks->purge_forced(old_hooks, addr, size, offset, length,
 	    arena_ind));
 }
 
@@ -158,8 +182,8 @@ extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
 	    "extent_hooks should be same as pointer used to set hooks");
 	assert_ptr_eq(extent_hooks->split, extent_split, "Wrong hook function");
 	tried_split = true;
-	err = old_hooks->split(old_hooks, addr, size, size_a, size_b, committed,
+	err = (old_hooks->split == NULL || old_hooks->split(old_hooks, addr,
-	    arena_ind);
+	    size, size_a, size_b, committed, arena_ind));
 	did_split = !err;
 	return (err);
 }
@@ -177,8 +201,8 @@ extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
 	assert_ptr_eq(extent_hooks, new_hooks,
 	    "extent_hooks should be same as pointer used to set hooks");
 	assert_ptr_eq(extent_hooks->merge, extent_merge, "Wrong hook function");
-	err = old_hooks->merge(old_hooks, addr_a, size_a, addr_b, size_b,
+	err = (old_hooks->merge == NULL || old_hooks->merge(old_hooks, addr_a,
-	    committed, arena_ind);
+	    size_a, addr_b, size_b, committed, arena_ind));
 	did_merge = !err;
 	return (err);
 }
@@ -216,7 +240,10 @@ TEST_BEGIN(test_extent)
 	    "Unexpected commit error");
 	assert_ptr_ne(old_hooks->decommit, extent_decommit,
 	    "Unexpected decommit error");
-	assert_ptr_ne(old_hooks->purge, extent_purge, "Unexpected purge error");
+	assert_ptr_ne(old_hooks->purge_lazy, extent_purge_lazy,
+	    "Unexpected purge_lazy error");
+	assert_ptr_ne(old_hooks->purge_forced, extent_purge_forced,
+	    "Unexpected purge_forced error");
 	assert_ptr_ne(old_hooks->split, extent_split, "Unexpected split error");
 	assert_ptr_ne(old_hooks->merge, extent_merge, "Unexpected merge error");
 
@@ -240,7 +267,8 @@ TEST_BEGIN(test_extent)
 	assert_ptr_not_null(p, "Unexpected mallocx() error");
 	did_dalloc = false;
 	did_decommit = false;
-	did_purge = false;
+	did_purge_lazy = false;
+	did_purge_forced = false;
 	tried_split = false;
 	did_split = false;
 	xallocx_success_a = (xallocx(p, large0, 0, flags) == large0);
@@ -249,7 +277,8 @@ TEST_BEGIN(test_extent)
 	if (xallocx_success_a) {
 		assert_true(did_dalloc, "Expected dalloc");
 		assert_false(did_decommit, "Unexpected decommit");
-		assert_true(did_purge, "Expected purge");
+		assert_true(did_purge_lazy || did_purge_forced,
+		    "Expected purge");
 	}
 	assert_true(tried_split, "Expected split");
 	dallocx(p, flags);
@@ -300,8 +329,10 @@ TEST_BEGIN(test_extent)
 	    "Unexpected commit error");
 	assert_ptr_eq(old_hooks->decommit, orig_hooks->decommit,
 	    "Unexpected decommit error");
-	assert_ptr_eq(old_hooks->purge, orig_hooks->purge,
+	assert_ptr_eq(old_hooks->purge_lazy, orig_hooks->purge_lazy,
-	    "Unexpected purge error");
+	    "Unexpected purge_lazy error");
+	assert_ptr_eq(old_hooks->purge_forced, orig_hooks->purge_forced,
+	    "Unexpected purge_forced error");
 	assert_ptr_eq(old_hooks->split, orig_hooks->split,
 	    "Unexpected split error");
 	assert_ptr_eq(old_hooks->merge, orig_hooks->merge,