Refactor huge_*() calls into arena internals.

Make redirects to the huge_*() API the arena code's responsibility,
since arenas now take responsibility for all allocation sizes.

include/jemalloc/internal/arena.h

@@ -391,7 +391,8 @@ void	arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info);
 void	arena_quarantine_junk_small(void *ptr, size_t usize);
 void	*arena_malloc_small(arena_t *arena, size_t size, bool zero);
 void	*arena_malloc_large(arena_t *arena, size_t size, bool zero);
-void	*arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
+void	*arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize,
+    size_t alignment, bool zero, tcache_t *tcache);
 void	arena_prof_promoted(const void *ptr, size_t size);
 void	arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk,
     void *ptr, arena_chunk_map_bits_t *bitselm);
@@ -481,8 +482,7 @@ void	*arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
 arena_t	*arena_aalloc(const void *ptr);
 size_t	arena_salloc(const void *ptr, bool demote);
 void	arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache);
-void	arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
+void	arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache);
-    tcache_t *tcache);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
@@ -931,20 +931,22 @@ arena_prof_tctx_get(const void *ptr)
 {
 	prof_tctx_t *ret;
 	arena_chunk_t *chunk;
-	size_t pageind, mapbits;
 
 	cassert(config_prof);
 	assert(ptr != NULL);
 	assert(CHUNK_ADDR2BASE(ptr) != ptr);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
+	if (likely(chunk != ptr)) {
-	mapbits = arena_mapbits_get(chunk, pageind);
+		size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
+		size_t mapbits = arena_mapbits_get(chunk, pageind);
 		assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
 		if (likely((mapbits & CHUNK_MAP_LARGE) == 0))
 			ret = (prof_tctx_t *)(uintptr_t)1U;
 		else
 			ret = arena_miscelm_get(chunk, pageind)->prof_tctx;
+	} else
+		ret = huge_prof_tctx_get(ptr);
 
 	return (ret);
 }
@@ -953,18 +955,20 @@ JEMALLOC_INLINE void
 arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
 {
 	arena_chunk_t *chunk;
-	size_t pageind;
 
 	cassert(config_prof);
 	assert(ptr != NULL);
 	assert(CHUNK_ADDR2BASE(ptr) != ptr);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
+	if (likely(chunk != ptr)) {
+		size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
 		assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
 
 		if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0))
 			arena_miscelm_get(chunk, pageind)->prof_tctx = tctx;
+	} else
+		huge_prof_tctx_set(ptr, tctx);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -984,7 +988,7 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
 				return (NULL);
 			return (arena_malloc_small(arena, size, zero));
 		}
-	} else {
+	} else if (likely(size <= arena_maxclass)) {
 		/*
 		 * Initialize tcache after checking size in order to avoid
 		 * infinite recursion during tcache initialization.
@@ -997,7 +1001,8 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
 				return (NULL);
 			return (arena_malloc_large(arena, size, zero));
 		}
-	}
+	} else
+		return (huge_malloc(tsd, arena, size, zero, tcache));
 }
 
 JEMALLOC_ALWAYS_INLINE arena_t *
@@ -1006,7 +1011,10 @@ arena_aalloc(const void *ptr)
 	arena_chunk_t *chunk;
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+	if (likely(chunk != ptr))
 		return (chunk->node.arena);
+	else
+		return (huge_aalloc(ptr));
 }
 
 /* Return the size of the allocation pointed to by ptr. */
@@ -1022,29 +1030,37 @@ arena_salloc(const void *ptr, bool demote)
 	assert(CHUNK_ADDR2BASE(ptr) != ptr);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+	if (likely(chunk != ptr)) {
 		pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
 		assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
 		binind = arena_mapbits_binind_get(chunk, pageind);
-	if (unlikely(binind == BININD_INVALID || (config_prof && !demote &&
+		if (unlikely(binind == BININD_INVALID || (config_prof && !demote
-	    arena_mapbits_large_get(chunk, pageind) != 0))) {
+		    && arena_mapbits_large_get(chunk, pageind) != 0))) {
 			/*
-		 * Large allocation.  In the common case (demote), and as this
+			 * Large allocation.  In the common case (demote), and
-		 * is an inline function, most callers will only end up looking
+			 * as this is an inline function, most callers will only
-		 * at binind to determine that ptr is a small allocation.
+			 * end up looking at binind to determine that ptr is a
+			 * small allocation.
 			 */
 			assert(((uintptr_t)ptr & PAGE_MASK) == 0);
 			ret = arena_mapbits_large_size_get(chunk, pageind);
 			assert(ret != 0);
 			assert(pageind + (ret>>LG_PAGE) <= chunk_npages);
 			assert(arena_mapbits_dirty_get(chunk, pageind) ==
-		    arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1));
+			    arena_mapbits_dirty_get(chunk,
+			    pageind+(ret>>LG_PAGE)-1));
 		} else {
-		/* Small allocation (possibly promoted to a large object). */
+			/*
+			 * Small allocation (possibly promoted to a large
+			 * object).
+			 */
 			assert(arena_mapbits_large_get(chunk, pageind) != 0 ||
-		    arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
+			    arena_ptr_small_binind_get(ptr,
-		    pageind)) == binind);
+			    arena_mapbits_get(chunk, pageind)) == binind);
 			ret = index2size(binind);
 		}
+	} else
+		ret = huge_salloc(ptr);
 
 	return (ret);
 }
@@ -1058,12 +1074,7 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
 	assert(ptr != NULL);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	if (unlikely(chunk == ptr)) {
+	if (likely(chunk != ptr)) {
-		huge_dalloc(tsd, ptr, tcache);
-		return;
-	}
-	assert(CHUNK_ADDR2BASE(ptr) != ptr);
-
 		pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
 		mapbits = arena_mapbits_get(chunk, pageind);
 		assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
@@ -1074,36 +1085,45 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
 				    mapbits);
 				tcache_dalloc_small(tsd, tcache, ptr, binind);
 			} else {
-			arena_dalloc_small(chunk->node.arena, chunk, ptr,
+				arena_dalloc_small(chunk->node.arena, chunk,
-			    pageind);
+				    ptr, pageind);
 			}
 		} else {
-		size_t size = arena_mapbits_large_size_get(chunk, pageind);
+			size_t size = arena_mapbits_large_size_get(chunk,
+			    pageind);
 
 			assert(((uintptr_t)ptr & PAGE_MASK) == 0);
 
 			if (likely(tcache != NULL) && size <= tcache_maxclass)
 				tcache_dalloc_large(tsd, tcache, ptr, size);
-		else
+			else {
-			arena_dalloc_large(chunk->node.arena, chunk, ptr);
+				arena_dalloc_large(chunk->node.arena, chunk,
+				    ptr);
 			}
 		}
+	} else
+		huge_dalloc(tsd, ptr, tcache);
+}
 
 JEMALLOC_ALWAYS_INLINE void
-arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
+arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
-    tcache_t *tcache)
 {
+	arena_chunk_t *chunk;
 
-	assert(ptr != NULL);
+	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	assert(CHUNK_ADDR2BASE(ptr) != ptr);
+	if (likely(chunk != ptr)) {
-
 		if (config_prof && opt_prof) {
-		size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
+			size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
+			    LG_PAGE;
 			assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
 			if (arena_mapbits_large_get(chunk, pageind) != 0) {
-			/* Make sure to use promoted size, not request size. */
+				/*
+				 * Make sure to use promoted size, not request
+				 * size.
+				 */
 				assert(((uintptr_t)ptr & PAGE_MASK) == 0);
-			size = arena_mapbits_large_size_get(chunk, pageind);
+				size = arena_mapbits_large_size_get(chunk,
+				    pageind);
 			}
 		}
 		assert(s2u(size) == s2u(arena_salloc(ptr, false)));
@@ -1114,20 +1134,24 @@ arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
 				index_t binind = size2index(size);
 				tcache_dalloc_small(tsd, tcache, ptr, binind);
 			} else {
-			size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
+				size_t pageind = ((uintptr_t)ptr -
-			    LG_PAGE;
+				    (uintptr_t)chunk) >> LG_PAGE;
-			arena_dalloc_small(chunk->node.arena, chunk, ptr,
+				arena_dalloc_small(chunk->node.arena, chunk,
-			    pageind);
+				    ptr, pageind);
 			}
 		} else {
 			assert(((uintptr_t)ptr & PAGE_MASK) == 0);
 
 			if (likely(tcache != NULL) && size <= tcache_maxclass)
 				tcache_dalloc_large(tsd, tcache, ptr, size);
-		else
+			else {
-			arena_dalloc_large(chunk->node.arena, chunk, ptr);
+				arena_dalloc_large(chunk->node.arena, chunk,
+				    ptr);
 			}
 		}
+	} else
+		huge_dalloc(tsd, ptr, tcache);
+}
 #  endif /* JEMALLOC_ARENA_INLINE_B */
 #endif
 

include/jemalloc/internal/jemalloc_internal.h.in

@@ -823,18 +823,10 @@ bool	ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 JEMALLOC_ALWAYS_INLINE arena_t *
 iaalloc(const void *ptr)
 {
-	arena_t *arena;
-	arena_chunk_t *chunk;
 
 	assert(ptr != NULL);
 
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+	return (arena_aalloc(ptr));
-	if (likely(chunk != ptr))
-		arena = arena_aalloc(ptr);
-	else
-		arena = huge_aalloc(ptr);
-
-	return (arena);
 }
 
 /*
@@ -845,20 +837,12 @@ iaalloc(const void *ptr)
 JEMALLOC_ALWAYS_INLINE size_t
 isalloc(const void *ptr, bool demote)
 {
-	size_t ret;
-	arena_chunk_t *chunk;
 
 	assert(ptr != NULL);
 	/* Demotion only makes sense if config_prof is true. */
 	assert(config_prof || !demote);
 
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+	return (arena_salloc(ptr, demote));
-	if (likely(chunk != ptr))
-		ret = arena_salloc(ptr, demote);
-	else
-		ret = huge_salloc(ptr);
-
-	return (ret);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -869,10 +853,7 @@ iallocztm(tsd_t *tsd, size_t size, bool zero, tcache_t *tcache, bool is_metadata
 
 	assert(size != 0);
 
-	if (likely(size <= arena_maxclass))
 	ret = arena_malloc(tsd, arena, size, zero, tcache);
-	else
-		ret = huge_malloc(tsd, arena, size, zero, tcache);
 	if (config_stats && is_metadata && likely(ret != NULL)) {
 		arena_metadata_allocated_add(iaalloc(ret), isalloc(ret,
 		    config_prof));
@@ -917,21 +898,7 @@ ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
 	assert(usize != 0);
 	assert(usize == sa2u(usize, alignment));
 
-	if (usize <= SMALL_MAXCLASS && alignment < PAGE)
+	ret = arena_palloc(tsd, arena, usize, alignment, zero, tcache);
-		ret = arena_malloc(tsd, arena, usize, zero, tcache);
-	else {
-		if (likely(usize <= arena_maxclass)) {
-			arena = arena_choose(tsd, arena);
-			if (unlikely(arena == NULL))
-				return (NULL);
-			ret = arena_palloc(arena, usize, alignment, zero);
-		} else if (likely(alignment <= chunksize))
-			ret = huge_malloc(tsd, arena, usize, zero, tcache);
-		else {
-			ret = huge_palloc(tsd, arena, usize, alignment, zero,
-			    tcache);
-		}
-	}
 	assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
 	if (config_stats && is_metadata && likely(ret != NULL)) {
 		arena_metadata_allocated_add(iaalloc(ret), isalloc(ret,
@@ -1033,15 +1000,8 @@ iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
 JEMALLOC_ALWAYS_INLINE void
 isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
 {
-	arena_chunk_t *chunk;
 
-	assert(ptr != NULL);
+	arena_sdalloc(tsd, ptr, size, tcache);
-
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	if (likely(chunk != ptr))
-		arena_sdalloc(tsd, chunk, ptr, size, tcache);
-	else
-		huge_dalloc(tsd, ptr, tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -1104,13 +1064,8 @@ iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
 		    zero, tcache, arena));
 	}
 
-	if (likely(size <= arena_maxclass)) {
+	return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0, alignment, zero,
-		return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0,
+	    tcache));
-		    alignment, zero, tcache));
-	} else {
-		return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0,
-		    alignment, zero, tcache));
-	}
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -1136,10 +1091,7 @@ ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment,
 		return (true);
 	}
 
-	if (likely(size <= arena_maxclass))
 	return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
-	else
-		return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero));
 }
 #endif
 

include/jemalloc/internal/prof.h

@@ -372,34 +372,21 @@ prof_tdata_get(tsd_t *tsd, bool create)
 JEMALLOC_ALWAYS_INLINE prof_tctx_t *
 prof_tctx_get(const void *ptr)
 {
-	prof_tctx_t *ret;
-	arena_chunk_t *chunk;
 
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+	return (arena_prof_tctx_get(ptr));
-	if (likely(chunk != ptr))
-		ret = arena_prof_tctx_get(ptr);
-	else
-		ret = huge_prof_tctx_get(ptr);
-
-	return (ret);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
 {
-	arena_chunk_t *chunk;
 
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	if (likely(chunk != ptr))
 	arena_prof_tctx_set(ptr, tctx);
-	else
-		huge_prof_tctx_set(ptr, tctx);
 }
 
 JEMALLOC_ALWAYS_INLINE bool

src/arena.c

@@ -1714,8 +1714,9 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero)
 }
 
 /* Only handles large allocations that require more than page alignment. */
-void *
+static void *
-arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
+arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
+    bool zero)
 {
 	void *ret;
 	size_t alloc_size, leadsize, trailsize;
@@ -1726,6 +1727,10 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
 
 	assert((size & PAGE_MASK) == 0);
 
+	arena = arena_choose(tsd, arena);
+	if (unlikely(arena == NULL))
+		return (NULL);
+
 	alignment = PAGE_CEILING(alignment);
 	alloc_size = size + alignment - PAGE;
 
@@ -1783,6 +1788,28 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
 	return (ret);
 }
 
+void *
+arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
+    bool zero, tcache_t *tcache)
+{
+	void *ret;
+
+	if (usize <= SMALL_MAXCLASS && alignment < PAGE)
+		ret = arena_malloc(tsd, arena, usize, zero, tcache);
+	else {
+		if (likely(usize <= arena_maxclass)) {
+			ret = arena_palloc_large(tsd, arena, usize, alignment,
+			    zero);
+		} else if (likely(alignment <= chunksize))
+			ret = huge_malloc(tsd, arena, usize, zero, tcache);
+		else {
+			ret = huge_palloc(tsd, arena, usize, alignment, zero,
+			    tcache);
+		}
+	}
+	return (ret);
+}
+
 void
 arena_prof_promoted(const void *ptr, size_t size)
 {
@@ -2189,22 +2216,26 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
     bool zero)
 {
 
+	if (likely(size <= arena_maxclass)) {
 		/*
-	 * Avoid moving the allocation if the size class can be left the same.
+		 * Avoid moving the allocation if the size class can be left the
+		 * same.
 		 */
 		if (likely(oldsize <= arena_maxclass)) {
 			if (oldsize <= SMALL_MAXCLASS) {
-			assert(arena_bin_info[size2index(oldsize)].reg_size
+				assert(
+				    arena_bin_info[size2index(oldsize)].reg_size
 				    == oldsize);
-			if ((size + extra <= SMALL_MAXCLASS && size2index(size +
+				if ((size + extra <= SMALL_MAXCLASS &&
-			    extra) == size2index(oldsize)) || (size <= oldsize
+				    size2index(size + extra) ==
-			    && size + extra >= oldsize))
+				    size2index(oldsize)) || (size <= oldsize &&
+				    size + extra >= oldsize))
 					return (false);
 			} else {
 				assert(size <= arena_maxclass);
 				if (size + extra > SMALL_MAXCLASS) {
-				if (!arena_ralloc_large(ptr, oldsize, size,
+					if (!arena_ralloc_large(ptr, oldsize,
-				    extra, zero))
+					    size, extra, zero))
 						return (false);
 				}
 			}
@@ -2212,6 +2243,8 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
 
 		/* Reallocation would require a move. */
 		return (true);
+	} else
+		return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero));
 }
 
 void *
@@ -2219,6 +2252,8 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
     size_t extra, size_t alignment, bool zero, tcache_t *tcache)
 {
 	void *ret;
+
+	if (likely(size <= arena_maxclass)) {
 		size_t copysize;
 
 		/* Try to avoid moving the allocation. */
@@ -2226,17 +2261,20 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
 			return (ptr);
 
 		/*
-	 * size and oldsize are different enough that we need to move the
+		 * size and oldsize are different enough that we need to move
-	 * object.  In that case, fall back to allocating new space and
+		 * the object.  In that case, fall back to allocating new space
-	 * copying.
+		 * and copying.
 		 */
 		if (alignment != 0) {
 			size_t usize = sa2u(size + extra, alignment);
 			if (usize == 0)
 				return (NULL);
-		ret = ipalloct(tsd, usize, alignment, zero, tcache, arena);
+			ret = ipalloct(tsd, usize, alignment, zero, tcache,
-	} else
+			    arena);
-		ret = arena_malloc(tsd, arena, size + extra, zero, tcache);
+		} else {
+			ret = arena_malloc(tsd, arena, size + extra, zero,
+			    tcache);
+		}
 
 		if (ret == NULL) {
 			if (extra == 0)
@@ -2246,25 +2284,35 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
 				size_t usize = sa2u(size, alignment);
 				if (usize == 0)
 					return (NULL);
-			ret = ipalloct(tsd, usize, alignment, zero, tcache,
+				ret = ipalloct(tsd, usize, alignment, zero,
-			    arena);
+				    tcache, arena);
-		} else
+			} else {
-			ret = arena_malloc(tsd, arena, size, zero, tcache);
+				ret = arena_malloc(tsd, arena, size, zero,
+				    tcache);
+			}
 
 			if (ret == NULL)
 				return (NULL);
 		}
 
-	/* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */
+		/*
+		 * Junk/zero-filling were already done by
+		 * ipalloc()/arena_malloc().
+		 */
 
 		/*
-	 * Copy at most size bytes (not size+extra), since the caller has no
+		 * Copy at most size bytes (not size+extra), since the caller
-	 * expectation that the extra bytes will be reliably preserved.
+		 * has no expectation that the extra bytes will be reliably
+		 * preserved.
 		 */
 		copysize = (size < oldsize) ? size : oldsize;
 		JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize);
 		memcpy(ret, ptr, copysize);
 		isqalloc(tsd, ptr, oldsize, tcache);
+	} else {
+		ret = huge_ralloc(tsd, arena, ptr, oldsize, size, extra,
+		    alignment, zero, tcache);
+	}
 	return (ret);
 }