Ehooks: Add a "zero" ehook.

This is the first API expansion.  It lets the hooks pick where and how to purge
within themselves.

include/jemalloc/internal/ehooks.h

@@ -43,6 +43,7 @@ bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
 #endif
 bool ehooks_default_split_impl();
 bool ehooks_default_merge_impl(void *addr_a, void *addr_b);
+void ehooks_default_zero_impl(void *addr, size_t size);
 
 /*
  * We don't officially support reentrancy from wtihin the extent hooks.  But
@@ -261,4 +262,21 @@ ehooks_merge(tsdn_t *tsdn, ehooks_t *ehooks, void *addr_a, size_t size_a,
 	}
 }
 
+static inline void
+ehooks_zero(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
+    unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks == &ehooks_default_extent_hooks) {
+		ehooks_default_zero_impl(addr, size);
+	} else {
+		/*
+		 * It would be correct to try using the user-provided purge
+		 * hooks (since they are required to have zeroed the extent if
+		 * they indicate success), but we don't necessarily know their
+		 * cost.  We'll be conservative and use memset.
+		 */
+		memset(addr, 0, size);
+	}
+}
+
 #endif /* JEMALLOC_INTERNAL_EHOOKS_H */

src/ehooks.c

@@ -209,6 +209,23 @@ ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
 	return ehooks_default_merge_impl(addr_a, addr_b);
 }
 
+void
+ehooks_default_zero_impl(void *addr, size_t size) {
+	/*
+	 * By default, we try to zero out memory using OS-provided demand-zeroed
+	 * pages.  If the user has specifically requested hugepages, though, we
+	 * don't want to purge in the middle of a hugepage (which would break it
+	 * up), so we act conservatively and use memset.
+	 */
+	bool needs_memset = true;
+	if (opt_thp != thp_mode_always) {
+		needs_memset = pages_purge_forced(addr, size);
+	}
+	if (needs_memset) {
+		memset(addr, 0, size);
+	}
+}
+
 const extent_hooks_t ehooks_default_extent_hooks = {
 	ehooks_default_alloc,
 	ehooks_default_dalloc,

src/extent2.c

@@ -758,17 +758,6 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 	unreachable();
 }
 
-static bool
-extent_need_manual_zero(arena_t *arena) {
-	/*
-	 * Need to manually zero the extent on repopulating if either; 1) non
-	 * default extent hooks installed (in which case the purge semantics may
-	 * change); or 2) transparent huge pages enabled.
-	 */
-	return (!ehooks_are_default(arena_get_ehooks(arena)) ||
-		(opt_thp == thp_mode_always));
-}
-
 /*
  * Tries to satisfy the given allocation request by reusing one of the extents
  * in the given eset_t.
@@ -807,9 +796,6 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
 			    growing_retained);
 			return NULL;
 		}
-		if (!extent_need_manual_zero(arena)) {
-			extent_zeroed_set(extent, true);
-		}
 	}
 
 	if (extent_committed_get(extent)) {
@@ -832,11 +818,10 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
 		void *addr = extent_base_get(extent);
 		if (!extent_zeroed_get(extent)) {
 			size_t size = extent_size_get(extent);
-			if (extent_need_manual_zero(arena) ||
+			ehooks_zero(tsdn, ehooks, addr, size,
-			    pages_purge_forced(addr, size)) {
+			    arena_ind_get(arena));
-				memset(addr, 0, size);
+		}
-			}
+		if (config_debug) {
-		} else if (config_debug) {
 			size_t *p = (size_t *)(uintptr_t)addr;
 			/* Check the first page only. */
 			for (size_t i = 0; i < PAGE / sizeof(size_t); i++) {
@@ -960,8 +945,14 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 			    &arena->eset_retained, extent, true);
 			goto label_err;
 		}
-		if (!extent_need_manual_zero(arena)) {
+		/* A successful commit should return zeroed memory. */
-			extent_zeroed_set(extent, true);
+		if (config_debug) {
+			void *addr = extent_addr_get(extent);
+			size_t *p = (size_t *)(uintptr_t)addr;
+			/* Check the first page only. */
+			for (size_t i = 0; i < PAGE / sizeof(size_t); i++) {
+				assert(p[i] == 0);
+			}
 		}
 	}
 
@@ -996,10 +987,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
 	if (*zero && !extent_zeroed_get(extent)) {
 		void *addr = extent_base_get(extent);
 		size_t size = extent_size_get(extent);
-		if (extent_need_manual_zero(arena) ||
+		ehooks_zero(tsdn, ehooks, addr, size, arena_ind_get(arena));
-		    pages_purge_forced(addr, size)) {
-			memset(addr, 0, size);
-		}
 	}
 
 	return extent;