Avoid abort() in test/integration/cpp/infallible_new_true.

Allow setting the safety check abort hook through mallctl, which avoids abort()
and core dumps.

include/jemalloc/internal/safety_check.h

@@ -4,8 +4,11 @@
 void safety_check_fail_sized_dealloc(bool current_dealloc, const void *ptr,
     size_t true_size, size_t input_size);
 void safety_check_fail(const char *format, ...);
+
+typedef void (*safety_check_abort_hook_t)(const char *message);
+
 /* Can set to NULL for a default. */
-void safety_check_set_abort(void (*abort_fn)(const char *));
+void safety_check_set_abort(safety_check_abort_hook_t abort_fn);
 
 JEMALLOC_ALWAYS_INLINE void
 safety_check_set_redzone(void *ptr, size_t usize, size_t bumped_usize) {

src/ctl.c

@@ -14,6 +14,7 @@
 #include "jemalloc/internal/prof_recent.h"
 #include "jemalloc/internal/prof_stats.h"
 #include "jemalloc/internal/prof_sys.h"
+#include "jemalloc/internal/safety_check.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/util.h"
 
@@ -311,6 +312,7 @@ CTL_PROTO(experimental_hooks_install)
 CTL_PROTO(experimental_hooks_remove)
 CTL_PROTO(experimental_hooks_prof_backtrace)
 CTL_PROTO(experimental_hooks_prof_dump)
+CTL_PROTO(experimental_hooks_safety_check_abort)
 CTL_PROTO(experimental_thread_activity_callback)
 CTL_PROTO(experimental_utilization_query)
 CTL_PROTO(experimental_utilization_batch_query)
@@ -849,6 +851,7 @@ static const ctl_named_node_t experimental_hooks_node[] = {
 	{NAME("remove"),	CTL(experimental_hooks_remove)},
 	{NAME("prof_backtrace"),	CTL(experimental_hooks_prof_backtrace)},
 	{NAME("prof_dump"),	CTL(experimental_hooks_prof_dump)},
+	{NAME("safety_check_abort"),	CTL(experimental_hooks_safety_check_abort)},
 };
 
 static const ctl_named_node_t experimental_thread_node[] = {
@@ -3437,6 +3440,27 @@ label_return:
 	return ret;
 }
 
+/* For integration test purpose only.  No plan to move out of experimental. */
+static int
+experimental_hooks_safety_check_abort_ctl(tsd_t *tsd, const size_t *mib,
+    size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
+	int ret;
+
+	WRITEONLY();
+	if (newp != NULL) {
+		if (newlen != sizeof(safety_check_abort_hook_t)) {
+			ret = EINVAL;
+			goto label_return;
+		}
+		safety_check_abort_hook_t hook JEMALLOC_CC_SILENCE_INIT(NULL);
+		WRITE(hook, safety_check_abort_hook_t);
+		safety_check_set_abort(hook);
+	}
+	ret = 0;
+label_return:
+	return ret;
+}
+
 /******************************************************************************/
 
 CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t)

src/safety_check.c

@@ -1,7 +1,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/jemalloc_internal_includes.h"
 
-static void (*safety_check_abort)(const char *message);
+static safety_check_abort_hook_t safety_check_abort;
 
 void safety_check_fail_sized_dealloc(bool current_dealloc, const void *ptr,
     size_t true_size, size_t input_size) {
@@ -15,7 +15,7 @@ void safety_check_fail_sized_dealloc(bool current_dealloc, const void *ptr,
 	    true_size, input_size, ptr, src);
 }
 
-void safety_check_set_abort(void (*abort_fn)(const char *)) {
+void safety_check_set_abort(safety_check_abort_hook_t abort_fn) {
 	safety_check_abort = abort_fn;
 }
 

test/integration/cpp/infallible_new_true.cpp

@@ -1,55 +1,61 @@
 #include <stdio.h>
 
-/*
- * We can't test C++ in unit tests, and we can't change the safety check failure
- * hook in integration tests.  So we check that we *actually* abort on failure,
- * by forking and checking the child process exit code.
- */
-
-/* It's a unix system? */
-#ifdef __unix__
-/* I know this! */
-#include <sys/types.h>
-#include <unistd.h>
-#include <sys/wait.h>
-static const bool can_fork = true;
-#else
-static const bool can_fork = false;
-#endif
-
 #include "test/jemalloc_test.h"
 
-TEST_BEGIN(test_failing_alloc) {
-	test_skip_if(!can_fork);
-#ifdef __unix__
-	pid_t pid = fork();
-	expect_d_ne(pid, -1, "Unexpected fork failure");
-	if (pid == 0) {
-		/*
-		 * In the child, we'll print an error message to stderr before
-		 * exiting.  Close stderr to avoid spamming output for this
-		 * expected failure.
-		 */
-		fclose(stderr);
-		try {
-			/* Too big of an allocation to succeed. */
-			void *volatile ptr = ::operator new((size_t)-1);
-			(void)ptr;
-		} catch (...) {
-			/*
-			 * Swallow the exception; remember, we expect this to
-			 * fail via an abort within new, not because an
-			 * exception didn't get caught.
-			 */
-		}
-	} else {
-		int status;
-		pid_t err = waitpid(pid, &status, 0);
-		expect_d_ne(-1, err, "waitpid failure");
-		expect_false(WIFEXITED(status),
-		    "Should have seen an abnormal failure");
+/*
+ * We can't test C++ in unit tests.  In order to intercept abort, use a secret
+ * safety check abort hook in integration tests.
+ */
+typedef void (*abort_hook_t)(const char *message);
+bool fake_abort_called;
+void fake_abort(const char *message) {
+	if (strcmp(message, "<jemalloc>: Allocation failed and "
+	    "opt.experimental_infallible_new is true. Aborting.\n") != 0) {
+		abort();
 	}
+	fake_abort_called = true;
+}
+
+static bool
+own_operator_new(void) {
+	uint64_t before, after;
+	size_t sz = sizeof(before);
+
+	/* thread.allocated is always available, even w/o config_stats. */
+	expect_d_eq(mallctl("thread.allocated", (void *)&before, &sz, NULL, 0),
+	    0, "Unexpected mallctl failure reading stats");
+	void *volatile ptr = ::operator new((size_t)8);
+	expect_ptr_not_null(ptr, "Unexpected allocation failure");
+	expect_d_eq(mallctl("thread.allocated", (void *)&after, &sz, NULL, 0),
+	    0, "Unexpected mallctl failure reading stats");
+
+	return (after != before);
+}
+
+TEST_BEGIN(test_failing_alloc) {
+	abort_hook_t abort_hook = &fake_abort;
+	expect_d_eq(mallctl("experimental.hooks.safety_check_abort", NULL, NULL,
+	    (void *)&abort_hook, sizeof(abort_hook)), 0,
+	    "Unexpected mallctl failure setting abort hook");
+
+	/*
+	 * Not owning operator new is only expected to happen on MinGW which
+	 * does not support operator new / delete replacement.
+	 */
+#ifdef _WIN32
+	test_skip_if(!own_operator_new());
+#else
+	expect_true(own_operator_new(), "No operator new overload");
 #endif
+	void *volatile ptr = (void *)1;
+	try {
+		/* Too big of an allocation to succeed. */
+		ptr = ::operator new((size_t)-1);
+	} catch (...) {
+		abort();
+	}
+	expect_ptr_null(ptr, "Allocation should have failed");
+	expect_b_eq(fake_abort_called, true, "Abort hook not invoked");
 }
 TEST_END