Refactor small_size2bin and small_bin2size.

Refactor small_size2bin and small_bin2size to be inline functions rather
than directly accessed arrays.

include/jemalloc/internal/arena.h

@@ -380,13 +380,17 @@ struct arena_s {
 
 extern ssize_t	opt_lg_dirty_mult;
 /*
- * small_size2bin is a compact lookup table that rounds request sizes up to
+ * small_size2bin_tab is a compact lookup table that rounds request sizes up to
  * size classes.  In order to reduce cache footprint, the table is compressed,
- * and all accesses are via the SMALL_SIZE2BIN macro.
+ * and all accesses are via small_size2bin().
  */
-extern uint8_t const	small_size2bin[];
+extern uint8_t const	small_size2bin_tab[];
-extern uint32_t const	small_bin2size[];
+/*
-#define	SMALL_SIZE2BIN(s)	(small_size2bin[(s-1) >> LG_TINY_MIN])
+ * small_bin2size_tab duplicates information in arena_bin_info, but in a const
+ * array, for which it is easier for the compiler to optimize repeated
+ * dereferences.
+ */
+extern uint32_t const	small_bin2size_tab[NBINS];
 
 extern arena_bin_info_t	arena_bin_info[NBINS];
 
@@ -450,6 +454,8 @@ void	arena_postfork_child(arena_t *arena);
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
+size_t	small_size2bin(size_t size);
+size_t	small_bin2size(size_t binind);
 arena_chunk_map_t	*arena_mapp_get(arena_chunk_t *chunk, size_t pageind);
 size_t	*arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbitsp_read(size_t *mapbitsp);
@@ -492,6 +498,22 @@ void	arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache);
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
 #  ifdef JEMALLOC_ARENA_INLINE_A
+JEMALLOC_ALWAYS_INLINE size_t
+small_size2bin(size_t size)
+{
+
+	return ((size_t)(small_size2bin_tab[(size-1) >> LG_TINY_MIN]));
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
+small_bin2size(size_t binind)
+{
+
+	return ((size_t)(small_bin2size_tab[binind]));
+}
+#  endif /* JEMALLOC_ARENA_INLINE_A */
+
+#  ifdef JEMALLOC_ARENA_INLINE_B
 JEMALLOC_ALWAYS_INLINE arena_chunk_map_t *
 arena_mapp_get(arena_chunk_t *chunk, size_t pageind)
 {
@@ -773,9 +795,9 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
 
 	return (binind);
 }
-#  endif /* JEMALLOC_ARENA_INLINE_A */
+#  endif /* JEMALLOC_ARENA_INLINE_B */
 
-#  ifdef JEMALLOC_ARENA_INLINE_B
+#  ifdef JEMALLOC_ARENA_INLINE_C
 JEMALLOC_INLINE size_t
 arena_bin_index(arena_t *arena, arena_bin_t *bin)
 {
@@ -965,7 +987,7 @@ arena_salloc(const void *ptr, bool demote)
 		assert(arena_mapbits_large_get(chunk, pageind) != 0 ||
 		    arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
 		    pageind)) == binind);
-		ret = small_bin2size[binind];
+		ret = small_bin2size(binind);
 	}
 
 	return (ret);
@@ -1004,7 +1026,7 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
 			arena_dalloc_large(chunk->arena, chunk, ptr);
 	}
 }
-#  endif /* JEMALLOC_ARENA_INLINE_B */
+#  endif /* JEMALLOC_ARENA_INLINE_C */
 #endif
 
 #endif /* JEMALLOC_H_INLINES */

include/jemalloc/internal/jemalloc_internal.h.in

@@ -499,6 +499,14 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/chunk.h"
 #include "jemalloc/internal/huge.h"
 
+/*
+ * Include arena.h the first time in order to provide inline functions for this
+ * header's inlines.
+ */
+#define	JEMALLOC_ARENA_INLINE_A
+#include "jemalloc/internal/arena.h"
+#undef JEMALLOC_ARENA_INLINE_A
+
 #ifndef JEMALLOC_ENABLE_INLINE
 malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
 
@@ -526,7 +534,7 @@ s2u(size_t size)
 {
 
 	if (size <= SMALL_MAXCLASS)
-		return (small_bin2size[SMALL_SIZE2BIN(size)]);
+		return (small_bin2size(small_size2bin(size)));
 	if (size <= arena_maxclass)
 		return (PAGE_CEILING(size));
 	return (CHUNK_CEILING(size));
@@ -569,7 +577,7 @@ sa2u(size_t size, size_t alignment)
 
 	if (usize <= arena_maxclass && alignment <= PAGE) {
 		if (usize <= SMALL_MAXCLASS)
-			return (small_bin2size[SMALL_SIZE2BIN(usize)]);
+			return (small_bin2size(small_size2bin(usize)));
 		return (PAGE_CEILING(usize));
 	} else {
 		size_t run_size;
@@ -643,16 +651,16 @@ choose_arena(arena_t *arena)
 #include "jemalloc/internal/bitmap.h"
 #include "jemalloc/internal/rtree.h"
 /*
- * Include arena.h twice in order to resolve circular dependencies with
+ * Include arena.h the second and third times in order to resolve circular
- * tcache.h.
+ * dependencies with tcache.h.
  */
-#define	JEMALLOC_ARENA_INLINE_A
-#include "jemalloc/internal/arena.h"
-#undef JEMALLOC_ARENA_INLINE_A
-#include "jemalloc/internal/tcache.h"
 #define	JEMALLOC_ARENA_INLINE_B
 #include "jemalloc/internal/arena.h"
 #undef JEMALLOC_ARENA_INLINE_B
+#include "jemalloc/internal/tcache.h"
+#define	JEMALLOC_ARENA_INLINE_C
+#include "jemalloc/internal/arena.h"
+#undef JEMALLOC_ARENA_INLINE_C
 #include "jemalloc/internal/hash.h"
 #include "jemalloc/internal/quarantine.h"
 
@@ -794,7 +802,7 @@ u2rz(size_t usize)
 	size_t ret;
 
 	if (usize <= SMALL_MAXCLASS) {
-		size_t binind = SMALL_SIZE2BIN(usize);
+		size_t binind = small_size2bin(usize);
 		ret = arena_bin_info[binind].redzone_size;
 	} else
 		ret = 0;

include/jemalloc/internal/private_symbols.txt

@@ -346,7 +346,9 @@ s2u
 sa2u
 set_errno
 small_bin2size
+small_bin2size_tab
 small_size2bin
+small_size2bin_tab
 stats_cactive
 stats_cactive_add
 stats_cactive_get

include/jemalloc/internal/tcache.h

@@ -263,10 +263,10 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 	size_t binind;
 	tcache_bin_t *tbin;
 
-	binind = SMALL_SIZE2BIN(size);
+	binind = small_size2bin(size);
 	assert(binind < NBINS);
 	tbin = &tcache->tbins[binind];
-	size = small_bin2size[binind];
+	size = small_bin2size(binind);
 	ret = tcache_alloc_easy(tbin);
 	if (ret == NULL) {
 		ret = tcache_alloc_small_hard(tcache, tbin, binind);

src/arena.c

@@ -8,7 +8,7 @@ ssize_t		opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT;
 arena_bin_info_t	arena_bin_info[NBINS];
 
 JEMALLOC_ALIGNED(CACHELINE)
-const uint32_t	small_bin2size[NBINS] = {
+const uint32_t	small_bin2size_tab[NBINS] = {
 #define SIZE_CLASS(bin, delta, size)		\
 	size,
 	SIZE_CLASSES
@@ -16,7 +16,7 @@ const uint32_t	small_bin2size[NBINS] = {
 };
 
 JEMALLOC_ALIGNED(CACHELINE)
-const uint8_t	small_size2bin[] = {
+const uint8_t	small_size2bin_tab[] = {
 #define	S2B_8(i)	i,
 #define	S2B_16(i)	S2B_8(i) S2B_8(i)
 #define	S2B_32(i)	S2B_16(i) S2B_16(i)
@@ -1607,7 +1607,7 @@ arena_quarantine_junk_small(void *ptr, size_t usize)
 	assert(opt_quarantine);
 	assert(usize <= SMALL_MAXCLASS);
 
-	binind = SMALL_SIZE2BIN(usize);
+	binind = small_size2bin(usize);
 	bin_info = &arena_bin_info[binind];
 	arena_redzones_validate(ptr, bin_info, true);
 }
@@ -1620,10 +1620,10 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 	arena_run_t *run;
 	size_t binind;
 
-	binind = SMALL_SIZE2BIN(size);
+	binind = small_size2bin(size);
 	assert(binind < NBINS);
 	bin = &arena->bins[binind];
-	size = small_bin2size[binind];
+	size = small_bin2size(binind);
 
 	malloc_mutex_lock(&bin->lock);
 	if ((run = bin->runcur) != NULL && run->nfree > 0)
@@ -1777,7 +1777,7 @@ arena_prof_promoted(const void *ptr, size_t size)
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-	binind = SMALL_SIZE2BIN(size);
+	binind = small_size2bin(size);
 	assert(binind < NBINS);
 	arena_mapbits_large_binind_set(chunk, pageind, binind);
 
@@ -2164,11 +2164,11 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
 	 */
 	if (oldsize <= arena_maxclass) {
 		if (oldsize <= SMALL_MAXCLASS) {
-			assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size
+			assert(arena_bin_info[small_size2bin(oldsize)].reg_size
 			    == oldsize);
 			if ((size + extra <= SMALL_MAXCLASS &&
-			    SMALL_SIZE2BIN(size + extra) ==
+			    small_size2bin(size + extra) ==
-			    SMALL_SIZE2BIN(oldsize)) || (size <= oldsize &&
+			    small_size2bin(oldsize)) || (size <= oldsize &&
 			    size + extra >= oldsize))
 				return (false);
 		} else {