Simplify arena_slab_regind().

Rewrite arena_slab_regind() to provide sufficient constant data for
the compiler to perform division strength reduction.  This replaces
more general manual strength reduction that was implemented before
arena_bin_info was compile-time-constant.  It would be possible to
slightly improve on the compiler-generated division code by taking
advantage of range limits that the compiler doesn't know about.

Makefile.in

@@ -184,6 +184,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/rtree.c \
 	$(srcroot)test/unit/SFMT.c \
 	$(srcroot)test/unit/size_classes.c \
+	$(srcroot)test/unit/slab.c \
 	$(srcroot)test/unit/smoothstep.c \
 	$(srcroot)test/unit/stats.c \
 	$(srcroot)test/unit/ticker.c \

include/jemalloc/internal/arena.h

@@ -271,6 +271,9 @@ void	arena_extent_cache_maybe_insert(tsdn_t *tsdn, arena_t *arena,
     extent_t *extent, bool cache);
 void	arena_extent_cache_maybe_remove(tsdn_t *tsdn, arena_t *arena,
     extent_t *extent, bool cache);
+#ifdef JEMALLOC_JET
+size_t	arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr);
+#endif
 extent_t	*arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena,
     size_t usize, size_t alignment, bool *zero);
 void	arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena,

include/jemalloc/internal/private_symbols.txt

@@ -66,6 +66,7 @@ arena_ralloc_no_move
 arena_reset
 arena_salloc
 arena_sdalloc
+arena_slab_regind
 arena_stats_merge
 arena_tcache_fill_small
 arena_tdata_get

src/arena.c

@@ -138,73 +138,41 @@ arena_slab_reg_alloc(tsdn_t *tsdn, extent_t *slab,
 	return (ret);
 }
 
-JEMALLOC_INLINE_C size_t
+#ifndef JEMALLOC_JET
-arena_slab_regind(extent_t *slab, const arena_bin_info_t *bin_info,
+JEMALLOC_INLINE_C
-    const void *ptr)
+#endif
+size_t
+arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr)
 {
-	size_t diff, interval, shift, regind;
+	size_t diff, regind;
 
 	/* Freeing a pointer outside the slab can cause assertion failure. */
 	assert((uintptr_t)ptr >= (uintptr_t)extent_addr_get(slab));
 	assert((uintptr_t)ptr < (uintptr_t)extent_past_get(slab));
 	/* Freeing an interior pointer can cause assertion failure. */
 	assert(((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)) %
-	    (uintptr_t)bin_info->reg_size == 0);
+	    (uintptr_t)arena_bin_info[binind].reg_size == 0);
 
-	/*
+	/* Avoid doing division with a variable divisor. */
-	 * Avoid doing division with a variable divisor if possible.  Using
-	 * actual division here can reduce allocator throughput by over 20%!
-	 */
 	diff = (size_t)((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab));
-
+	switch (binind) {
-	/* Rescale (factor powers of 2 out of the numerator and denominator). */
+#define	REGIND_bin_yes(index, reg_size)					\
-	interval = bin_info->reg_size;
+	case index:							\
-	shift = ffs_zu(interval) - 1;
+		regind = diff / (reg_size);				\
-	diff >>= shift;
+		assert(diff == regind * (reg_size));			\
-	interval >>= shift;
+		break;
-
+#define	REGIND_bin_no(index, reg_size)
-	if (interval == 1) {
+#define	SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs,		\
-		/* The divisor was a power of 2. */
+    lg_delta_lookup)							\
-		regind = diff;
+	REGIND_bin_##bin(index, (1U<<lg_grp) + (ndelta<<lg_delta))
-	} else {
+	SIZE_CLASSES
-		/*
+#undef REGIND_bin_yes
-		 * To divide by a number D that is not a power of two we
+#undef REGIND_bin_no
-		 * multiply by (2^21 / D) and then right shift by 21 positions.
+#undef SC
-		 *
+	default: not_reached();
-		 *   X / D
-		 *
-		 * becomes
-		 *
-		 *   (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT
-		 *
-		 * We can omit the first three elements, because we never
-		 * divide by 0, and 1 and 2 are both powers of two, which are
-		 * handled above.
-		 */
-#define	SIZE_INV_SHIFT	((sizeof(size_t) << 3) - LG_SLAB_MAXREGS)
-#define	SIZE_INV(s)	(((ZU(1) << SIZE_INV_SHIFT) / (s)) + 1)
-		static const size_t interval_invs[] = {
-		    SIZE_INV(3),
-		    SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
-		    SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
-		    SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
-		    SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
-		    SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
-		    SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
-		    SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
-		};
-
-		if (likely(interval <= ((sizeof(interval_invs) / sizeof(size_t))
-		    + 2))) {
-			regind = (diff * interval_invs[interval - 3]) >>
-			    SIZE_INV_SHIFT;
-		} else
-			regind = diff / interval;
-#undef SIZE_INV
-#undef SIZE_INV_SHIFT
 	}
-	assert(diff == regind * interval);
+
-	assert(regind < bin_info->nregs);
+	assert(regind < arena_bin_info[binind].nregs);
 
 	return (regind);
 }
@@ -215,7 +183,7 @@ arena_slab_reg_dalloc(tsdn_t *tsdn, extent_t *slab,
 {
 	szind_t binind = slab_data->binind;
 	const arena_bin_info_t *bin_info = &arena_bin_info[binind];
-	size_t regind = arena_slab_regind(slab, bin_info, ptr);
+	size_t regind = arena_slab_regind(slab, binind, ptr);
 
 	assert(slab_data->nfree < bin_info->nregs);
 	/* Freeing an unallocated pointer can cause assertion failure. */
@@ -1022,7 +990,6 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin,
 	const arena_bin_info_t *bin_info;
 	extent_t *slab;
 
-
 	bin_info = &arena_bin_info[binind];
 	if (bin->slabcur != NULL) {
 		arena_bin_slabs_full_insert(bin, bin->slabcur);

test/unit/slab.c

@@ -0,0 +1,35 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_arena_slab_regind)
+{
+	szind_t binind;
+
+	for (binind = 0; binind < NBINS; binind++) {
+		size_t regind;
+		extent_t slab;
+		const arena_bin_info_t *bin_info = &arena_bin_info[binind];
+		extent_init(&slab, NULL, mallocx(bin_info->slab_size,
+		    MALLOCX_LG_ALIGN(LG_PAGE)), bin_info->slab_size, 0, 0, true,
+		    false, true, true);
+		assert_ptr_not_null(extent_addr_get(&slab),
+		    "Unexpected malloc() failure");
+		for (regind = 0; regind < bin_info->nregs; regind++) {
+			void *reg = (void *)((uintptr_t)extent_addr_get(&slab) +
+			    (bin_info->reg_size * regind));
+			assert_zu_eq(arena_slab_regind(&slab, binind, reg),
+			    regind,
+			    "Incorrect region index computed for size %zu",
+			    bin_info->reg_size);
+		}
+		free(extent_addr_get(&slab));
+	}
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_arena_slab_regind));
+}