Make arena_salloc() an inline function.

include/jemalloc/internal/arena.h

@@ -408,7 +408,6 @@ void	arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info);
 void	*arena_malloc_small(arena_t *arena, size_t size, bool zero);
 void	*arena_malloc_large(arena_t *arena, size_t size, bool zero);
 void	*arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
-size_t	arena_salloc(const void *ptr, bool demote);
 void	arena_prof_promoted(const void *ptr, size_t size);
 void	arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
     arena_chunk_map_t *mapelm);
@@ -437,6 +436,7 @@ unsigned	arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
 prof_ctx_t	*arena_prof_ctx_get(const void *ptr);
 void	arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
 void	*arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache);
+size_t	arena_salloc(const void *ptr, bool demote);
 void	arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr,
     bool try_tcache);
 #endif
@@ -625,6 +625,46 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
 	}
 }
 
+/* Return the size of the allocation pointed to by ptr. */
+JEMALLOC_INLINE size_t
+arena_salloc(const void *ptr, bool demote)
+{
+	size_t ret;
+	arena_chunk_t *chunk;
+	size_t pageind, mapbits;
+
+	assert(ptr != NULL);
+	assert(CHUNK_ADDR2BASE(ptr) != ptr);
+
+	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
+	mapbits = chunk->map[pageind-map_bias].bits;
+	assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
+	if ((mapbits & CHUNK_MAP_LARGE) == 0) {
+		arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
+		    (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE));
+		size_t binind = arena_bin_index(chunk->arena, run->bin);
+		arena_bin_info_t *bin_info = &arena_bin_info[binind];
+		assert(((uintptr_t)ptr - ((uintptr_t)run +
+		    (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval
+		    == 0);
+		ret = bin_info->reg_size;
+	} else {
+		assert(((uintptr_t)ptr & PAGE_MASK) == 0);
+		ret = mapbits & ~PAGE_MASK;
+		if (config_prof && demote && prof_promote && ret == PAGE &&
+		    (mapbits & CHUNK_MAP_CLASS_MASK) != 0) {
+			size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >>
+			    CHUNK_MAP_CLASS_SHIFT) - 1;
+			assert(binind < NBINS);
+			ret = arena_bin_info[binind].reg_size;
+		}
+		assert(ret != 0);
+	}
+
+	return (ret);
+}
+
 JEMALLOC_INLINE void
 arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
 {

include/jemalloc/internal/jemalloc_internal.h.in

@@ -732,10 +732,9 @@ isalloc(const void *ptr, bool demote)
 	assert(config_prof || demote == false);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	if (chunk != ptr) {
+	if (chunk != ptr)
-		/* Region. */
 		ret = arena_salloc(ptr, demote);
-	} else
+	else
 		ret = huge_salloc(ptr);
 
 	return (ret);

include/jemalloc/internal/tcache.h

@@ -100,6 +100,9 @@ extern size_t			nhbins;
 /* Maximum cached size class. */
 extern size_t			tcache_maxclass;
 
+size_t	tcache_salloc(const void *ptr);
+void	*tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin,
+    size_t binind);
 void	tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
     tcache_t *tcache);
 void	tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
@@ -107,8 +110,6 @@ void	tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
 void	tcache_arena_associate(tcache_t *tcache, arena_t *arena);
 void	tcache_arena_dissociate(tcache_t *tcache);
 tcache_t *tcache_create(arena_t *arena);
-void	*tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin,
-    size_t binind);
 void	tcache_destroy(tcache_t *tcache);
 void	tcache_thread_cleanup(void *arg);
 void	tcache_stats_merge(tcache_t *tcache, arena_t *arena);
@@ -340,7 +341,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 		if (ret == NULL)
 			return (NULL);
 	}
-	assert(arena_salloc(ret, false) == arena_bin_info[binind].reg_size);
+	assert(tcache_salloc(ret) == arena_bin_info[binind].reg_size);
 
 	if (zero == false) {
 		if (config_fill) {
@@ -431,7 +432,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr)
 	size_t pageind, binind;
 	arena_chunk_map_t *mapelm;
 
-	assert(arena_salloc(ptr, false) <= SMALL_MAXCLASS);
+	assert(tcache_salloc(ptr) <= SMALL_MAXCLASS);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	arena = chunk->arena;
@@ -468,8 +469,8 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
 	tcache_bin_info_t *tbin_info;
 
 	assert((size & PAGE_MASK) == 0);
-	assert(arena_salloc(ptr, false) > SMALL_MAXCLASS);
+	assert(tcache_salloc(ptr) > SMALL_MAXCLASS);
-	assert(arena_salloc(ptr, false) <= tcache_maxclass);
+	assert(tcache_salloc(ptr) <= tcache_maxclass);
 
 	binind = NBINS + (size >> LG_PAGE) - 1;
 

src/arena.c

@@ -1465,46 +1465,6 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
 	return (ret);
 }
 
-/* Return the size of the allocation pointed to by ptr. */
-size_t
-arena_salloc(const void *ptr, bool demote)
-{
-	size_t ret;
-	arena_chunk_t *chunk;
-	size_t pageind, mapbits;
-
-	assert(ptr != NULL);
-	assert(CHUNK_ADDR2BASE(ptr) != ptr);
-
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-	mapbits = chunk->map[pageind-map_bias].bits;
-	assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
-	if ((mapbits & CHUNK_MAP_LARGE) == 0) {
-		arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
-		    (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE));
-		size_t binind = arena_bin_index(chunk->arena, run->bin);
-		arena_bin_info_t *bin_info = &arena_bin_info[binind];
-		assert(((uintptr_t)ptr - ((uintptr_t)run +
-		    (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval
-		    == 0);
-		ret = bin_info->reg_size;
-	} else {
-		assert(((uintptr_t)ptr & PAGE_MASK) == 0);
-		ret = mapbits & ~PAGE_MASK;
-		if (config_prof && demote && prof_promote && ret == PAGE &&
-		    (mapbits & CHUNK_MAP_CLASS_MASK) != 0) {
-			size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >>
-			    CHUNK_MAP_CLASS_SHIFT) - 1;
-			assert(binind < NBINS);
-			ret = arena_bin_info[binind].reg_size;
-		}
-		assert(ret != 0);
-	}
-
-	return (ret);
-}
-
 void
 arena_prof_promoted(const void *ptr, size_t size)
 {

src/tcache.c

@@ -18,6 +18,12 @@ size_t			tcache_maxclass;
 
 /******************************************************************************/
 
+size_t	tcache_salloc(const void *ptr)
+{
+
+	return (arena_salloc(ptr, false));
+}
+
 void *
 tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
 {