Refactor purging and splitting/merging.

Split purging into lazy and forced variants. Use the forced variant for zeroing dss. Add support for NULL function pointers as an opt-out mechanism for the dalloc, commit, decommit, purge_lazy, purge_forced, split, and merge fields of extent_hooks_t. Add short-circuiting checks in large_ralloc_no_move_{shrink,expand}() so that no attempt is made if splitting/merging is not supported. This resolves #268.
2016-12-03 15:38:25 -08:00
parent 884fa22b8c
commit a6e86810d8
11 changed files with 258 additions and 84 deletions
--- a/src/extent.c
+++ b/src/extent.c
@@ -15,23 +15,47 @@ static bool	extent_commit_default(extent_hooks_t *extent_hooks, void *addr,
    size_t size, size_t offset, size_t length, unsigned arena_ind);
 static bool	extent_decommit_default(extent_hooks_t *extent_hooks,
    void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
-static bool	extent_purge_default(extent_hooks_t *extent_hooks, void *addr,
-    size_t size, size_t offset, size_t length, unsigned arena_ind);
+#ifdef PAGES_CAN_PURGE_LAZY
+static bool	extent_purge_lazy_default(extent_hooks_t *extent_hooks,
+    void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
+#endif
+#ifdef PAGES_CAN_PURGE_FORCED
+static bool	extent_purge_forced_default(extent_hooks_t *extent_hooks,
+    void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
+#endif
+#ifdef JEMALLOC_MAPS_COALESCE
 static bool	extent_split_default(extent_hooks_t *extent_hooks, void *addr,
    size_t size, size_t size_a, size_t size_b, bool committed,
    unsigned arena_ind);
 static bool	extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a,
    size_t size_a, void *addr_b, size_t size_b, bool committed,
    unsigned arena_ind);
+#endif

 const extent_hooks_t	extent_hooks_default = {
 	extent_alloc_default,
 	extent_dalloc_default,
 	extent_commit_default,
-	extent_decommit_default,
-	extent_purge_default,
+	extent_decommit_default
+#ifdef PAGES_CAN_PURGE_LAZY
+	,
+	extent_purge_lazy_default
+#else
+	,
+	NULL
+#endif
+#ifdef PAGES_CAN_PURGE_FORCED
+	,
+	extent_purge_forced_default
+#else
+	,
+	NULL
+#endif
+#ifdef JEMALLOC_MAPS_COALESCE
+	,
 	extent_split_default,
 	extent_merge_default
+#endif
 };

 /* Used exclusively for gdump triggering. */
@@ -395,8 +419,11 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
 	 * that this is only a virtual memory leak.
 	 */
 	if (cache) {
-		extent_purge_wrapper(tsdn, arena, r_extent_hooks, extent, 0,
-		    extent_size_get(extent));
+		if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks,
+		    extent, 0, extent_size_get(extent))) {
+			extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks,
+			    extent, 0, extent_size_get(extent));
+		}
 	}
 	extent_dalloc(tsdn, arena, extent);
 }
@@ -1023,7 +1050,7 @@ void
 extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
    extent_hooks_t **r_extent_hooks, extent_t *extent)
 {
-	bool err;
+	bool err, zeroed;

 	assert(extent_base_get(extent) != NULL);
 	assert(extent_size_get(extent) != 0);
@@ -1041,9 +1068,10 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
 		err = extent_dalloc_default_impl(extent_base_get(extent),
 		    extent_size_get(extent));
 	} else {
-		err = (*r_extent_hooks)->dalloc(*r_extent_hooks,
+		err = ((*r_extent_hooks)->dalloc == NULL ||
+		    (*r_extent_hooks)->dalloc(*r_extent_hooks,
 		    extent_base_get(extent), extent_size_get(extent),
-		    extent_committed_get(extent), arena->ind);
+		    extent_committed_get(extent), arena->ind));
 	}

 	if (!err) {
@@ -1052,13 +1080,24 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
 	}
 	extent_reregister(tsdn, extent);
 	/* Try to decommit; purge if that fails. */
-	if (extent_committed_get(extent)) {
-		extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
-		    0, extent_size_get(extent));
-	}
-	extent_zeroed_set(extent, !extent_committed_get(extent) ||
-	    !(*r_extent_hooks)->purge(*r_extent_hooks, extent_base_get(extent),
-	    extent_size_get(extent), 0, extent_size_get(extent), arena->ind));
+	if (!extent_committed_get(extent))
+		zeroed = true;
+	else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
+	    0, extent_size_get(extent)))
+		zeroed = true;
+	else if ((*r_extent_hooks)->purge_lazy != NULL &&
+	    !(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
+	    extent_base_get(extent), extent_size_get(extent), 0,
+	    extent_size_get(extent), arena->ind))
+		zeroed = false;
+	else if ((*r_extent_hooks)->purge_forced != NULL &&
+	    !(*r_extent_hooks)->purge_forced(*r_extent_hooks,
+	    extent_base_get(extent), extent_size_get(extent), 0,
+	    extent_size_get(extent), arena->ind))
+		zeroed = true;
+	else
+		zeroed = false;
+	extent_zeroed_set(extent, zeroed);

 	if (config_stats)
 		arena->stats.retained += extent_size_get(extent);
@@ -1088,9 +1127,9 @@ extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
 	bool err;

 	extent_hooks_assure_initialized(arena, r_extent_hooks);
-	err = (*r_extent_hooks)->commit(*r_extent_hooks,
-	    extent_base_get(extent), extent_size_get(extent), offset, length,
-	    arena->ind);
+	err = ((*r_extent_hooks)->commit == NULL ||
+	    (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent),
+	    extent_size_get(extent), offset, length, arena->ind));
 	extent_committed_set(extent, extent_committed_get(extent) || !err);
 	return (err);
 }
@@ -1115,15 +1154,17 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,

 	extent_hooks_assure_initialized(arena, r_extent_hooks);

-	err = (*r_extent_hooks)->decommit(*r_extent_hooks,
+	err = ((*r_extent_hooks)->decommit == NULL ||
+	    (*r_extent_hooks)->decommit(*r_extent_hooks,
 	    extent_base_get(extent), extent_size_get(extent), offset, length,
-	    arena->ind);
+	    arena->ind));
 	extent_committed_set(extent, extent_committed_get(extent) && err);
 	return (err);
 }

+#ifdef PAGES_CAN_PURGE_LAZY
 static bool
-extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
+extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
    size_t offset, size_t length, unsigned arena_ind)
 {

@@ -1133,22 +1174,55 @@ extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
 	assert(length != 0);
 	assert((length & PAGE_MASK) == 0);

-	return (pages_purge((void *)((uintptr_t)addr + (uintptr_t)offset),
+	return (pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset),
 	    length));
 }
+#endif

 bool
-extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
+extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
    size_t length)
 {

 	extent_hooks_assure_initialized(arena, r_extent_hooks);
-	return ((*r_extent_hooks)->purge(*r_extent_hooks,
+	return ((*r_extent_hooks)->purge_lazy == NULL ||
+	    (*r_extent_hooks)->purge_lazy(*r_extent_hooks,
 	    extent_base_get(extent), extent_size_get(extent), offset, length,
 	    arena->ind));
 }

+#ifdef PAGES_CAN_PURGE_FORCED
+static bool
+extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, size_t offset, size_t length, unsigned arena_ind)
+{
+
+	assert(extent_hooks == &extent_hooks_default);
+	assert(addr != NULL);
+	assert((offset & PAGE_MASK) == 0);
+	assert(length != 0);
+	assert((length & PAGE_MASK) == 0);
+
+	return (pages_purge_forced((void *)((uintptr_t)addr +
+	    (uintptr_t)offset), length));
+}
+#endif
+
+bool
+extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
+    size_t length)
+{
+
+	extent_hooks_assure_initialized(arena, r_extent_hooks);
+	return ((*r_extent_hooks)->purge_forced == NULL ||
+	    (*r_extent_hooks)->purge_forced(*r_extent_hooks,
+	    extent_base_get(extent), extent_size_get(extent), offset, length,
+	    arena->ind));
+}
+
+#ifdef JEMALLOC_MAPS_COALESCE
 static bool
 extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
    size_t size_a, size_t size_b, bool committed, unsigned arena_ind)
@@ -1160,6 +1234,7 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
 		return (true);
 	return (false);
 }
+#endif

 extent_t *
 extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
@@ -1175,6 +1250,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,

 	extent_hooks_assure_initialized(arena, r_extent_hooks);

+	if ((*r_extent_hooks)->split == NULL)
+		return (NULL);
+
 	trail = extent_alloc(tsdn, arena);
 	if (trail == NULL)
 		goto label_error_a;
@@ -1237,6 +1315,7 @@ extent_merge_default_impl(void *addr_a, void *addr_b)
 	return (false);
 }

+#ifdef JEMALLOC_MAPS_COALESCE
 static bool
 extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
    void *addr_b, size_t size_b, bool committed, unsigned arena_ind)
@@ -1246,6 +1325,7 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,

 	return (extent_merge_default_impl(addr_a, addr_b));
 }
+#endif

 bool
 extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
@@ -1257,6 +1337,10 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
 	rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;

 	extent_hooks_assure_initialized(arena, r_extent_hooks);
+
+	if ((*r_extent_hooks)->merge == NULL)
+		return (true);
+
 	if (*r_extent_hooks == &extent_hooks_default) {
 		/* Call directly to propagate tsdn. */
 		err = extent_merge_default_impl(extent_base_get(a),
--- a/src/extent_dss.c
+++ b/src/extent_dss.c
@@ -168,10 +168,20 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
 					extent_dalloc_gap(tsdn, arena, gap);
 				else
 					extent_dalloc(tsdn, arena, gap);
-				if (*zero)
-					memset(ret, 0, size);
 				if (!*commit)
 					*commit = pages_decommit(ret, size);
+				if (*zero && *commit) {
+					extent_hooks_t *extent_hooks =
+					    EXTENT_HOOKS_INITIALIZER;
+					extent_t extent;
+
+					extent_init(&extent, arena, ret, size,
+					    size, 0, true, false, true, false);
+					if (extent_purge_forced_wrapper(tsdn,
+					    arena, &extent_hooks, &extent, 0,
+					    size))
+						memset(ret, 0, size);
+				}
 				return (ret);
 			}
 			/*
--- a/src/large.c
+++ b/src/large.c
@@ -110,6 +110,9 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)

 	assert(oldusize > usize);

+	if (extent_hooks->split == NULL)
+		return (true);
+
 	/* Split excess pages. */
 	if (diff != 0) {
 		extent_t *trail = extent_split_wrapper(tsdn, arena,
@@ -142,6 +145,9 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
 	size_t trailsize = usize - extent_usize_get(extent);
 	extent_t *trail;

+	if (extent_hooks->merge == NULL)
+		return (true);
+
 	if ((trail = arena_extent_cache_alloc(tsdn, arena, &extent_hooks,
 	    extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail)) ==
 	    NULL) {
--- a/src/pages.c
+++ b/src/pages.c
@@ -163,33 +163,34 @@ pages_decommit(void *addr, size_t size)
 }

 bool
-pages_purge(void *addr, size_t size)
+pages_purge_lazy(void *addr, size_t size)
 {
-	bool unzeroed;
+
+	if (!pages_can_purge_lazy)
+		return (true);

 #ifdef _WIN32
 	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
-	unzeroed = true;
-#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \
-    defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
-#  if defined(JEMALLOC_PURGE_MADVISE_FREE)
-#    define JEMALLOC_MADV_PURGE MADV_FREE
-#    define JEMALLOC_MADV_ZEROS false
-#  elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
-#    define JEMALLOC_MADV_PURGE MADV_DONTNEED
-#    define JEMALLOC_MADV_ZEROS true
-#  else
-#    error No madvise(2) flag defined for purging unused dirty pages
-#  endif
-	int err = madvise(addr, size, JEMALLOC_MADV_PURGE);
-	unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0);
-#  undef JEMALLOC_MADV_PURGE
-#  undef JEMALLOC_MADV_ZEROS
+#elif defined(JEMALLOC_PURGE_MADVISE_FREE)
+	madvise(addr, size, MADV_FREE);
 #else
-	/* Last resort no-op. */
-	unzeroed = true;
+	not_reached();
+#endif
+	return (false);
+}
+
+bool
+pages_purge_forced(void *addr, size_t size)
+{
+
+	if (!pages_can_purge_forced)
+		return (true);
+
+#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
+	return (madvise(addr, size, MADV_DONTNEED) != 0);
+#else
+	not_reached();
 #endif
-	return (unzeroed);
 }

 bool