Extract profiling code from [re]allocation functions.

Extract profiling code from malloc(), imemalign(), calloc(), realloc(),
mallocx(), rallocx(), and xallocx().  This slightly reduces the amount
of code compiled into the fast paths, but the primary benefit is the
combinatorial complexity reduction.

Simplify iralloc[t]() by creating a separate ixalloc() that handles the
no-move cases.

Further simplify [mrxn]allocx() (and by implication [mrn]allocm()) to
make request size overflows due to size class and/or alignment
constraints trigger undefined behavior (detected by debug-only
assertions).

Report ENOMEM rather than EINVAL if an OOM occurs during heap profiling
backtrace creation in imemalign().  This bug impacted posix_memalign()
and aligned_alloc().

doc/jemalloc.xml.in

@@ -321,14 +321,16 @@
       <para>The <function>mallocx<parameter/></function> function allocates at
       least <parameter>size</parameter> bytes of memory, and returns a pointer
       to the base address of the allocation.  Behavior is undefined if
-      <parameter>size</parameter> is <constant>0</constant>.</para>
+      <parameter>size</parameter> is <constant>0</constant>, or if request size
+      overflows due to size class and/or alignment constraints.</para>
 
       <para>The <function>rallocx<parameter/></function> function resizes the
       allocation at <parameter>ptr</parameter> to be at least
       <parameter>size</parameter> bytes, and returns a pointer to the base
       address of the resulting allocation, which may or may not have moved from
       its original location.  Behavior is undefined if
-      <parameter>size</parameter> is <constant>0</constant>.</para>
+      <parameter>size</parameter> is <constant>0</constant>, or if request size
+      overflows due to size class and/or alignment constraints.</para>
 
       <para>The <function>xallocx<parameter/></function> function resizes the
       allocation at <parameter>ptr</parameter> in place to be at least
@@ -355,8 +357,9 @@
       <function>mallocx<parameter/></function> function, and returns the real
       size of the allocation that would result from the equivalent
       <function>mallocx<parameter/></function> function call.  Behavior is
-      undefined if <parameter>size</parameter> is
+      undefined if <parameter>size</parameter> is <constant>0</constant>, or if
-      <constant>0</constant>.</para>
+      request size overflows due to size class and/or alignment
+      constraints.</para>
 
       <para>The <function>mallctl<parameter/></function> function provides a
       general interface for introspecting the memory allocator, as well as
@@ -518,8 +521,9 @@ for (i = 0; i < nbins; i++) {
       <parameter>*ptr</parameter> to the base address of the allocation, and
       sets <parameter>*rsize</parameter> to the real size of the allocation if
       <parameter>rsize</parameter> is not <constant>NULL</constant>.  Behavior
-      is undefined if <parameter>size</parameter> is
+      is undefined if <parameter>size</parameter> is <constant>0</constant>, or
-      <constant>0</constant>.</para>
+      if request size overflows due to size class and/or alignment
+      constraints.</para>
 
       <para>The <function>rallocm<parameter/></function> function resizes the
       allocation at <parameter>*ptr</parameter> to be at least
@@ -532,8 +536,9 @@ for (i = 0; i < nbins; i++) {
       language="C">(<parameter>size</parameter> +
       <parameter>extra</parameter>)</code> bytes, though inability to allocate
       the extra byte(s) will not by itself result in failure.  Behavior is
-      undefined if <parameter>size</parameter> is <constant>0</constant>, or if
+      undefined if <parameter>size</parameter> is <constant>0</constant>, if
-      <code language="C">(<parameter>size</parameter> +
+      request size overflows due to size class and/or alignment constraints, or
+      if <code language="C">(<parameter>size</parameter> +
       <parameter>extra</parameter> &gt;
       <constant>SIZE_T_MAX</constant>)</code>.</para>
 
@@ -550,8 +555,9 @@ for (i = 0; i < nbins; i++) {
       <parameter>rsize</parameter> is not <constant>NULL</constant> it sets
       <parameter>*rsize</parameter> to the real size of the allocation that
       would result from the equivalent <function>allocm<parameter/></function>
-      function call.  Behavior is undefined if
+      function call.  Behavior is undefined if <parameter>size</parameter> is
-      <parameter>size</parameter> is <constant>0</constant>.</para>
+      <constant>0</constant>, or if request size overflows due to size class
+      and/or alignment constraints.</para>
     </refsect2>
   </refsect1>
   <refsect1 id="tuning">

include/jemalloc/internal/arena.h

@@ -436,7 +436,7 @@ void	arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
 typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t);
 extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
 #endif
-void	*arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
+bool	arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra, bool zero);
 void	*arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
     size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,

include/jemalloc/internal/huge.h

@@ -19,7 +19,7 @@ extern malloc_mutex_t	huge_mtx;
 
 void	*huge_malloc(size_t size, bool zero);
 void	*huge_palloc(size_t size, size_t alignment, bool zero);
-void	*huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
+bool	huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra);
 void	*huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
     size_t alignment, bool zero, bool try_tcache_dalloc);

include/jemalloc/internal/jemalloc_internal.h.in

@@ -747,11 +747,15 @@ void	idalloct(void *ptr, bool try_tcache);
 void	idalloc(void *ptr);
 void	iqalloct(void *ptr, bool try_tcache);
 void	iqalloc(void *ptr);
-void	*iralloct(void *ptr, size_t size, size_t extra, size_t alignment,
+void	*iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
-    bool zero, bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc,
+    size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
     arena_t *arena);
+void	*iralloct(void *ptr, size_t size, size_t extra, size_t alignment,
+    bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
 void	*iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
-    bool zero, bool no_move);
+    bool zero);
+bool	ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
+    bool zero);
 malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
 #endif
 
@@ -920,10 +924,42 @@ iqalloc(void *ptr)
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
+iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
-    bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
+    size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
+    arena_t *arena)
+{
+	void *p;
+	size_t usize, copysize;
+
+	usize = sa2u(size + extra, alignment);
+	if (usize == 0)
+		return (NULL);
+	p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
+	if (p == NULL) {
+		if (extra == 0)
+			return (NULL);
+		/* Try again, without extra this time. */
+		usize = sa2u(size, alignment);
+		if (usize == 0)
+			return (NULL);
+		p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
+		if (p == NULL)
+			return (NULL);
+	}
+	/*
+	 * Copy at most size bytes (not size+extra), since the caller has no
+	 * expectation that the extra bytes will be reliably preserved.
+	 */
+	copysize = (size < oldsize) ? size : oldsize;
+	memcpy(p, ptr, copysize);
+	iqalloct(ptr, try_tcache_dalloc);
+	return (p);
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
+    bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
 {
-	void *ret;
 	size_t oldsize;
 
 	assert(ptr != NULL);
@@ -933,68 +969,50 @@ iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 
 	if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
 	    != 0) {
-		size_t usize, copysize;
-
 		/*
 		 * Existing object alignment is inadequate; allocate new space
 		 * and copy.
 		 */
-		if (no_move)
+		return (iralloct_realign(ptr, oldsize, size, extra, alignment,
-			return (NULL);
+		    zero, try_tcache_alloc, try_tcache_dalloc, arena));
-		usize = sa2u(size + extra, alignment);
-		if (usize == 0)
-			return (NULL);
-		ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
-		if (ret == NULL) {
-			if (extra == 0)
-				return (NULL);
-			/* Try again, without extra this time. */
-			usize = sa2u(size, alignment);
-			if (usize == 0)
-				return (NULL);
-			ret = ipalloct(usize, alignment, zero, try_tcache_alloc,
-			    arena);
-			if (ret == NULL)
-				return (NULL);
-		}
-		/*
-		 * Copy at most size bytes (not size+extra), since the caller
-		 * has no expectation that the extra bytes will be reliably
-		 * preserved.
-		 */
-		copysize = (size < oldsize) ? size : oldsize;
-		memcpy(ret, ptr, copysize);
-		iqalloct(ptr, try_tcache_dalloc);
-		return (ret);
 	}
 
-	if (no_move) {
+	if (size + extra <= arena_maxclass) {
-		if (size <= arena_maxclass) {
+		return (arena_ralloc(arena, ptr, oldsize, size, extra,
-			return (arena_ralloc_no_move(ptr, oldsize, size,
+		    alignment, zero, try_tcache_alloc,
-			    extra, zero));
+		    try_tcache_dalloc));
-		} else {
-			return (huge_ralloc_no_move(ptr, oldsize, size,
-			    extra));
-		}
 	} else {
-		if (size + extra <= arena_maxclass) {
+		return (huge_ralloc(ptr, oldsize, size, extra,
-			return (arena_ralloc(arena, ptr, oldsize, size, extra,
+		    alignment, zero, try_tcache_dalloc));
-			    alignment, zero, try_tcache_alloc,
-			    try_tcache_dalloc));
-		} else {
-			return (huge_ralloc(ptr, oldsize, size, extra,
-			    alignment, zero, try_tcache_dalloc));
-		}
 	}
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
+iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
-    bool no_move)
 {
 
-	return (iralloct(ptr, size, extra, alignment, zero, no_move, true, true,
+	return (iralloct(ptr, size, extra, alignment, zero, true, true, NULL));
-	    NULL));
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
+{
+	size_t oldsize;
+
+	assert(ptr != NULL);
+	assert(size != 0);
+
+	oldsize = isalloc(ptr, config_prof);
+	if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
+	    != 0) {
+		/* Existing object alignment is inadequate. */
+		return (true);
+	}
+
+	if (size <= arena_maxclass)
+		return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
+	else
+		return (huge_ralloc_no_move(ptr, oldsize, size, extra));
 }
 
 malloc_tsd_externs(thread_allocated, thread_allocated_t)

include/jemalloc/internal/private_symbols.txt

@@ -223,9 +223,11 @@ iqalloc
 iqalloct
 iralloc
 iralloct
+iralloct_realign
 isalloc
 isthreaded
 ivsalloc
+ixalloc
 jemalloc_postfork_child
 jemalloc_postfork_parent
 jemalloc_prefork

src/arena.c

@@ -2061,7 +2061,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra,
 	}
 }
 
-void *
+bool
 arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
     bool zero)
 {
@@ -2077,19 +2077,19 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
 			    SMALL_SIZE2BIN(size + extra) ==
 			    SMALL_SIZE2BIN(oldsize)) || (size <= oldsize &&
 			    size + extra >= oldsize))
-				return (ptr);
+				return (false);
 		} else {
 			assert(size <= arena_maxclass);
 			if (size + extra > SMALL_MAXCLASS) {
 				if (arena_ralloc_large(ptr, oldsize, size,
 				    extra, zero) == false)
-					return (ptr);
+					return (false);
 			}
 		}
 	}
 
 	/* Reallocation would require a move. */
-	return (NULL);
+	return (true);
 }
 
 void *
@@ -2101,9 +2101,8 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
 	size_t copysize;
 
 	/* Try to avoid moving the allocation. */
-	ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero);
+	if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false)
-	if (ret != NULL)
+		return (ptr);
-		return (ret);
 
 	/*
 	 * size and oldsize are different enough that we need to move the

src/huge.c

@@ -78,7 +78,7 @@ huge_palloc(size_t size, size_t alignment, bool zero)
 	return (ret);
 }
 
-void *
+bool
 huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
 {
 
@@ -89,11 +89,11 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
 	    && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
 	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
 		assert(CHUNK_CEILING(oldsize) == oldsize);
-		return (ptr);
+		return (false);
 	}
 
 	/* Reallocation would require a move. */
-	return (NULL);
+	return (true);
 }
 
 void *
@@ -104,9 +104,8 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 	size_t copysize;
 
 	/* Try to avoid moving the allocation. */
-	ret = huge_ralloc_no_move(ptr, oldsize, size, extra);
+	if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false)
-	if (ret != NULL)
+		return (ptr);
-		return (ret);
 
 	/*
 	 * size and oldsize are different enough that we need to use a

test/integration/allocm.c

@@ -44,23 +44,6 @@ TEST_BEGIN(test_alignment_errors)
 	void *p;
 	size_t nsz, rsz, sz, alignment;
 
-#if LG_SIZEOF_PTR == 3
-	alignment = UINT64_C(0x8000000000000000);
-	sz        = UINT64_C(0x8000000000000000);
-#else
-	alignment = 0x80000000LU;
-	sz        = 0x80000000LU;
-#endif
-	nsz = 0;
-	assert_d_ne(nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)), ALLOCM_SUCCESS,
-	    "Expected error for nallocm(&nsz, %zu, %#x)",
-	    sz, ALLOCM_ALIGN(alignment));
-	rsz = 0;
-	assert_d_ne(allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)),
-	    ALLOCM_SUCCESS, "Expected error for allocm(&p, %zu, %#x)",
-	    sz, ALLOCM_ALIGN(alignment));
-	assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch");
-
 #if LG_SIZEOF_PTR == 3
 	alignment = UINT64_C(0x4000000000000000);
 	sz        = UINT64_C(0x8400000000000001);
@@ -75,22 +58,6 @@ TEST_BEGIN(test_alignment_errors)
 	assert_d_ne(allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)),
 	    ALLOCM_SUCCESS, "Expected error for allocm(&p, %zu, %#x)",
 	    sz, ALLOCM_ALIGN(alignment));
-
-	alignment = 0x10LU;
-#if LG_SIZEOF_PTR == 3
-	sz = UINT64_C(0xfffffffffffffff0);
-#else
-	sz = 0xfffffff0LU;
-#endif
-	nsz = 0;
-	assert_d_ne(nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)), ALLOCM_SUCCESS,
-	    "Expected error for nallocm(&nsz, %zu, %#x)",
-	    sz, ALLOCM_ALIGN(alignment));
-	rsz = 0;
-	assert_d_ne(allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)),
-	    ALLOCM_SUCCESS, "Expected error for allocm(&p, %zu, %#x)",
-	    sz, ALLOCM_ALIGN(alignment));
-	assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch");
 }
 TEST_END
 

test/integration/mallocx.c

@@ -39,20 +39,6 @@ TEST_BEGIN(test_alignment_errors)
 	void *p;
 	size_t nsz, sz, alignment;
 
-#if LG_SIZEOF_PTR == 3
-	alignment = UINT64_C(0x8000000000000000);
-	sz        = UINT64_C(0x8000000000000000);
-#else
-	alignment = 0x80000000LU;
-	sz        = 0x80000000LU;
-#endif
-	nsz = nallocx(sz, MALLOCX_ALIGN(alignment));
-	assert_zu_eq(nsz, 0, "Expected error for nallocx(%zu, %#x)", sz,
-	    MALLOCX_ALIGN(alignment));
-	p = mallocx(sz, MALLOCX_ALIGN(alignment));
-	assert_ptr_null(p, "Expected error for mallocx(%zu, %#x)", sz,
-	    MALLOCX_ALIGN(alignment));
-
 #if LG_SIZEOF_PTR == 3
 	alignment = UINT64_C(0x4000000000000000);
 	sz        = UINT64_C(0x8400000000000001);
@@ -65,22 +51,6 @@ TEST_BEGIN(test_alignment_errors)
 	p = mallocx(sz, MALLOCX_ALIGN(alignment));
 	assert_ptr_null(p, "Expected error for mallocx(%zu, %#x)", sz,
 	    MALLOCX_ALIGN(alignment));
-
-	alignment = 0x10LU;
-#if LG_SIZEOF_PTR == 3
-	sz = UINT64_C(0xfffffffffffffff0);
-#else
-	sz = 0xfffffff0LU;
-#endif
-	nsz = nallocx(sz, MALLOCX_ALIGN(alignment));
-	assert_zu_eq(nsz, 0, "Expected error for nallocx(%zu, %#x)", sz,
-	    MALLOCX_ALIGN(alignment));
-	nsz = nallocx(sz, MALLOCX_ALIGN(alignment));
-	assert_zu_eq(nsz, 0, "Expected error for nallocx(%zu, %#x)", sz,
-	    MALLOCX_ALIGN(alignment));
-	p = mallocx(sz, MALLOCX_ALIGN(alignment));
-	assert_ptr_null(p, "Expected error for mallocx(%zu, %#x)", sz,
-	    MALLOCX_ALIGN(alignment));
 }
 TEST_END